You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/09/20 20:30:09 UTC
[01/50] [abbrv] hadoop git commit: [JDK10] Upgrade Maven Javadoc
Plugin from 3.0.0-M1 to 3.0.1.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 c377e3ca4 -> 77e106f74
[JDK10] Upgrade Maven Javadoc Plugin from 3.0.0-M1 to 3.0.1.
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/281c192e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/281c192e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/281c192e
Branch: refs/heads/HDFS-12943
Commit: 281c192e7f917545151117fc7e067dc480b93499
Parents: 8b2f5e6
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Tue Sep 18 09:31:47 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Sep 18 09:32:27 2018 +0900
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/281c192e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 20a2723..3669ffb 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1628,7 +1628,7 @@
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
<configuration>
- <additionalparam>-Xmaxwarns 10000</additionalparam>
+ <additionalOptions>-Xmaxwarns 10000</additionalOptions>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/281c192e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 73d5cf5..5780a79 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
<maven-dependency-plugin.version>3.0.2</maven-dependency-plugin.version>
<maven-enforcer-plugin.version>3.0.0-M1</maven-enforcer-plugin.version>
- <maven-javadoc-plugin.version>3.0.0-M1</maven-javadoc-plugin.version>
+ <maven-javadoc-plugin.version>3.0.1</maven-javadoc-plugin.version>
<maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
<maven-remote-resources-plugin.version>1.5</maven-remote-resources-plugin.version>
<maven-resources-plugin.version>3.0.1</maven-resources-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDDS-440. Datanode loops forever
if it cannot create directories. Contributed by Bharat Viswanadham.
Posted by xk...@apache.org.
HDDS-440. Datanode loops forever if it cannot create directories.
Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a968ea48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a968ea48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a968ea48
Branch: refs/heads/HDFS-12943
Commit: a968ea489743ed09d63a6e267e34491e490cd2d8
Parents: e71f61e
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Sep 18 14:31:50 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Sep 18 14:31:50 2018 -0700
----------------------------------------------------------------------
.../states/datanode/InitDatanodeState.java | 12 +++++-
.../common/TestDatanodeStateMachine.java | 42 ++++++++++++++++++++
2 files changed, 52 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a968ea48/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
index b348327..995f172 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
@@ -116,7 +116,7 @@ public class InitDatanodeState implements DatanodeState,
/**
* Persist DatanodeDetails to datanode.id file.
*/
- private void persistContainerDatanodeDetails() throws IOException {
+ private void persistContainerDatanodeDetails() {
String dataNodeIDPath = HddsUtils.getDatanodeIdFilePath(conf);
if (Strings.isNullOrEmpty(dataNodeIDPath)) {
LOG.error("A valid file path is needed for config setting {}",
@@ -128,7 +128,15 @@ public class InitDatanodeState implements DatanodeState,
DatanodeDetails datanodeDetails = this.context.getParent()
.getDatanodeDetails();
if (datanodeDetails != null && !idPath.exists()) {
- ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
+ try {
+ ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
+ } catch (IOException ex) {
+ // As writing DatanodeDetails in to datanodeid file failed, which is
+ // a critical thing, so shutting down the state machine.
+ LOG.error("Writing to {} failed {}", dataNodeIDPath, ex.getMessage());
+ this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
+ return;
+ }
LOG.info("DatanodeDetails is persisted to {}", dataNodeIDPath);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a968ea48/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 59029db..3fc0dd0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -311,6 +311,48 @@ public class TestDatanodeStateMachine {
}
}
+ @Test
+ public void testDatanodeStateMachineWithIdWriteFail() throws Exception {
+
+ File idPath = new File(
+ conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID));
+ idPath.delete();
+ DatanodeDetails datanodeDetails = getNewDatanodeDetails();
+ DatanodeDetails.Port port = DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.STANDALONE,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ datanodeDetails.setPort(port);
+
+ try (DatanodeStateMachine stateMachine =
+ new DatanodeStateMachine(datanodeDetails, conf)) {
+ DatanodeStateMachine.DatanodeStates currentState =
+ stateMachine.getContext().getState();
+ Assert.assertEquals(DatanodeStateMachine.DatanodeStates.INIT,
+ currentState);
+
+ DatanodeState<DatanodeStateMachine.DatanodeStates> task =
+ stateMachine.getContext().getTask();
+ Assert.assertEquals(InitDatanodeState.class, task.getClass());
+
+ //Set the idPath to read only, state machine will fail to write
+ // datanodeId file and set the state to shutdown.
+ idPath.getParentFile().mkdirs();
+ idPath.getParentFile().setReadOnly();
+
+ task.execute(executorService);
+ DatanodeStateMachine.DatanodeStates newState =
+ task.await(2, TimeUnit.SECONDS);
+
+ //As, we have changed the permission of idPath to readable, writing
+ // will fail and it will set the state to shutdown.
+ Assert.assertEquals(DatanodeStateMachine.DatanodeStates.SHUTDOWN,
+ newState);
+
+ //Setting back to writable.
+ idPath.getParentFile().setWritable(true);
+ }
+ }
+
/**
* Test state transition with a list of invalid scm configurations,
* and verify the state transits to SHUTDOWN each time.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-491. Minor typos in README.md
in smoketest. Contributed by chencan.
Posted by xk...@apache.org.
HDDS-491. Minor typos in README.md in smoketest. Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51fda2d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51fda2d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51fda2d7
Branch: refs/heads/HDFS-12943
Commit: 51fda2d7733a17a22f68c1c57b0ada062b713620
Parents: ee051ef
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Sep 17 22:22:56 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Sep 17 22:22:56 2018 -0700
----------------------------------------------------------------------
hadoop-dist/src/main/smoketest/README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51fda2d7/hadoop-dist/src/main/smoketest/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/README.md b/hadoop-dist/src/main/smoketest/README.md
index cc6ee92..c521a54 100644
--- a/hadoop-dist/src/main/smoketest/README.md
+++ b/hadoop-dist/src/main/smoketest/README.md
@@ -20,11 +20,11 @@ You can run in in any environment after [installing](https://github.com/robotfra
```
cd $DIRECTORY_OF_OZONE
-robot smoketest/bascic
+robot smoketest/basic
```
The argument of the `robot` could be any robot file or directory.
The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment.
-The `./test.sh` in this directory can start multiple type of clusters (ozone standalon or ozone + hdfs) and execute the test framework with all of the clusters.
\ No newline at end of file
+The `./test.sh` in this directory can start multiple type of clusters (ozone standalone or ozone + hdfs) and execute the test framework with all of the clusters.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDFS-13886.
HttpFSFileSystem.getFileStatus() doesn't return "snapshot enabled" bit.
Contributed by Siyao Meng.
Posted by xk...@apache.org.
HDFS-13886. HttpFSFileSystem.getFileStatus() doesn't return "snapshot enabled" bit. Contributed by Siyao Meng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44857476
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44857476
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44857476
Branch: refs/heads/HDFS-12943
Commit: 44857476fa993fbf9c97f979b91e19d27632c10a
Parents: 8382b86
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Sep 18 15:33:02 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Sep 18 15:33:02 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/http/client/HttpFSFileSystem.java | 2 +-
.../hadoop/fs/http/server/FSOperations.java | 3 ++
.../fs/http/client/BaseTestHttpFSWith.java | 35 +++++++++++++++++++-
3 files changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44857476/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index ce76f05..dd285d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -199,7 +199,7 @@ public class HttpFSFileSystem extends FileSystem
public static final String ENC_BIT_JSON = "encBit";
public static final String EC_BIT_JSON = "ecBit";
- public static final String SNAPSHOT_BIT_JSON = "seBit";
+ public static final String SNAPSHOT_BIT_JSON = "snapshotEnabled";
public static final String DIRECTORY_LISTING_JSON = "DirectoryListing";
public static final String PARTIAL_LISTING_JSON = "partialListing";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44857476/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index 1d47a61..a3c45c79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -120,6 +120,9 @@ public class FSOperations {
if (fileStatus.getPermission().getErasureCodedBit()) {
json.put(HttpFSFileSystem.EC_BIT_JSON, true);
}
+ if (fileStatus.isSnapshotEnabled()) {
+ json.put(HttpFSFileSystem.SNAPSHOT_BIT_JSON, true);
+ }
return json;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44857476/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index a6dce4d..8dabdea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -376,6 +376,35 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
Assert.assertEquals(stati[0].getPath(), statl[0].getPath());
}
+ private void testFileStatusAttr() throws Exception {
+ if (!this.isLocalFS()) {
+ // Create a directory
+ Path path = new Path("/tmp/tmp-snap-test");
+ DistributedFileSystem distributedFs = (DistributedFileSystem) FileSystem
+ .get(path.toUri(), this.getProxiedFSConf());
+ distributedFs.mkdirs(path);
+ // Get the FileSystem instance that's being tested
+ FileSystem fs = this.getHttpFSFileSystem();
+ // Check FileStatus
+ assertFalse("Snapshot should be disallowed by default",
+ fs.getFileStatus(path).isSnapshotEnabled());
+ // Allow snapshot
+ distributedFs.allowSnapshot(path);
+ // Check FileStatus
+ assertTrue("Snapshot enabled bit is not set in FileStatus",
+ fs.getFileStatus(path).isSnapshotEnabled());
+ // Disallow snapshot
+ distributedFs.disallowSnapshot(path);
+ // Check FileStatus
+ assertFalse("Snapshot enabled bit is not cleared in FileStatus",
+ fs.getFileStatus(path).isSnapshotEnabled());
+ // Cleanup
+ fs.delete(path, true);
+ fs.close();
+ distributedFs.close();
+ }
+ }
+
private static void assertSameListing(FileSystem expected, FileSystem
actual, Path p) throws IOException {
// Consume all the entries from both iterators
@@ -1041,7 +1070,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR,
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH,
GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING,
- CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT
+ CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT,
+ FILE_STATUS_ATTR
}
private void operation(Operation op) throws Exception {
@@ -1139,6 +1169,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
case DELETE_SNAPSHOT:
testDeleteSnapshot();
break;
+ case FILE_STATUS_ATTR:
+ testFileStatusAttr();
+ break;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-15772. Remove the 'Path ...
should be specified as a URI' warnings on startup. Contributed by Ayush
Saxena.
Posted by xk...@apache.org.
HADOOP-15772. Remove the 'Path ... should be specified as a URI' warnings on startup. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2eb597b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2eb597b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2eb597b1
Branch: refs/heads/HDFS-12943
Commit: 2eb597b1511f8f46866abe4eeec820f4191cc295
Parents: a30b4f9
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Sep 19 13:48:27 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Sep 19 13:48:27 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/server/common/Util.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2eb597b1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index 4e30e50..e9f9bfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -97,8 +97,7 @@ public final class Util {
// if URI is null or scheme is undefined, then assume it's file://
if(u == null || u.getScheme() == null){
- LOG.warn("Path " + s + " should be specified as a URI "
- + "in configuration files. Please update hdfs configuration.");
+ LOG.info("Assuming 'file' scheme for path " + s + " in configuration.");
u = fileAsURI(new File(s));
}
return u;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDDS-513. Check if the EventQueue
is not closed before executing handlers. Contributed by Nanda Kumar.
Posted by xk...@apache.org.
HDDS-513. Check if the EventQueue is not closed before executing handlers. Contributed by Nanda Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6bb1ca3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6bb1ca3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6bb1ca3
Branch: refs/heads/HDFS-12943
Commit: f6bb1ca3c16eb5825188b51d45f32be111266d88
Parents: 2eb597b
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Sep 19 14:35:29 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Sep 19 14:35:29 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/server/events/EventQueue.java | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6bb1ca3/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
index b2b0df2..9aeab7b 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventQueue.java
@@ -55,6 +55,8 @@ public class EventQueue implements EventPublisher, AutoCloseable {
private final AtomicLong eventCount = new AtomicLong(0);
+ private boolean isRunning = true;
+
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void addHandler(
EVENT_TYPE event, EventHandler<PAYLOAD> handler) {
this.addHandler(event, handler, generateHandlerName(handler));
@@ -116,6 +118,10 @@ public class EventQueue implements EventPublisher, AutoCloseable {
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void addHandler(
EVENT_TYPE event, EventExecutor<PAYLOAD> executor,
EventHandler<PAYLOAD> handler) {
+ if (!isRunning) {
+ LOG.warn("Not adding handler for {}, EventQueue is not running", event);
+ return;
+ }
validateEvent(event);
executors.putIfAbsent(event, new HashMap<>());
executors.get(event).putIfAbsent(executor, new ArrayList<>());
@@ -136,6 +142,11 @@ public class EventQueue implements EventPublisher, AutoCloseable {
public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
EVENT_TYPE event, PAYLOAD payload) {
+ if (!isRunning) {
+ LOG.warn("Processing of {} is skipped, EventQueue is not running", event);
+ return;
+ }
+
Map<EventExecutor, List<EventHandler>> eventExecutorListMap =
this.executors.get(event);
@@ -187,6 +198,11 @@ public class EventQueue implements EventPublisher, AutoCloseable {
long currentTime = Time.now();
while (true) {
+ if (!isRunning) {
+ LOG.warn("Processing of event skipped. EventQueue is not running");
+ return;
+ }
+
long processed = 0;
Stream<EventExecutor> allExecutor = this.executors.values().stream()
@@ -216,6 +232,8 @@ public class EventQueue implements EventPublisher, AutoCloseable {
public void close() {
+ isRunning = false;
+
Set<EventExecutor> allExecutors = this.executors.values().stream()
.flatMap(handlerMap -> handlerMap.keySet().stream())
.collect(Collectors.toSet());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-476. Add Pipeline reports to
make pipeline active on SCM restart. Contributed by Mukul Kumar Singh.
Posted by xk...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
index 82946bd..59d937e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
@@ -16,9 +16,12 @@
*/
package org.apache.hadoop.hdds.scm.pipelines;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
@@ -30,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
import org.apache.hadoop.hdds.scm.pipelines.ratis.RatisManagerImpl;
import org.apache.hadoop.hdds.scm.pipelines.standalone.StandaloneManagerImpl;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -75,11 +79,9 @@ public class PipelineSelector {
private static final Logger LOG =
LoggerFactory.getLogger(PipelineSelector.class);
private final ContainerPlacementPolicy placementPolicy;
- private final NodeManager nodeManager;
+ private final Map<ReplicationType, PipelineManager> pipelineManagerMap;
private final Configuration conf;
private final EventPublisher eventPublisher;
- private final RatisManagerImpl ratisManager;
- private final StandaloneManagerImpl standaloneManager;
private final long containerSize;
private final MetadataStore pipelineStore;
private final PipelineStateManager stateManager;
@@ -96,7 +98,6 @@ public class PipelineSelector {
*/
public PipelineSelector(NodeManager nodeManager, Configuration conf,
EventPublisher eventPublisher, int cacheSizeMB) throws IOException {
- this.nodeManager = nodeManager;
this.conf = conf;
this.eventPublisher = eventPublisher;
this.placementPolicy = createContainerPlacementPolicy(nodeManager, conf);
@@ -106,12 +107,14 @@ public class PipelineSelector {
StorageUnit.BYTES);
node2PipelineMap = new Node2PipelineMap();
pipelineMap = new ConcurrentHashMap<>();
- this.standaloneManager =
- new StandaloneManagerImpl(this.nodeManager, placementPolicy,
- containerSize);
- this.ratisManager =
- new RatisManagerImpl(this.nodeManager, placementPolicy, containerSize,
- conf);
+ pipelineManagerMap = new HashMap<>();
+
+ pipelineManagerMap.put(ReplicationType.STAND_ALONE,
+ new StandaloneManagerImpl(nodeManager, placementPolicy,
+ containerSize));
+ pipelineManagerMap.put(ReplicationType.RATIS,
+ new RatisManagerImpl(nodeManager, placementPolicy,
+ containerSize, conf));
long pipelineCreationLeaseTimeout = conf.getTimeDuration(
ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT,
ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT_DEFAULT,
@@ -154,6 +157,7 @@ public class PipelineSelector {
}
}
+ @VisibleForTesting
public Set<ContainerID> getOpenContainerIDsByPipeline(PipelineID pipelineID) {
return pipeline2ContainerMap.get(pipelineID);
}
@@ -227,30 +231,6 @@ public class PipelineSelector {
}
/**
- * Return the pipeline manager from the replication type.
- *
- * @param replicationType - Replication Type Enum.
- * @return pipeline Manager.
- * @throws IllegalArgumentException If an pipeline type gets added
- * and this function is not modified we will throw.
- */
- private PipelineManager getPipelineManager(ReplicationType replicationType)
- throws IllegalArgumentException {
- switch (replicationType) {
- case RATIS:
- return this.ratisManager;
- case STAND_ALONE:
- return this.standaloneManager;
- case CHAINED:
- throw new IllegalArgumentException("Not implemented yet");
- default:
- throw new IllegalArgumentException("Unexpected enum found. Does not" +
- " know how to handle " + replicationType.toString());
- }
-
- }
-
- /**
* This function is called by the Container Manager while allocating a new
* container. The client specifies what kind of replication pipeline is needed
* and based on the replication type in the request appropriate Interface is
@@ -260,7 +240,7 @@ public class PipelineSelector {
public Pipeline getReplicationPipeline(ReplicationType replicationType,
HddsProtos.ReplicationFactor replicationFactor)
throws IOException {
- PipelineManager manager = getPipelineManager(replicationType);
+ PipelineManager manager = pipelineManagerMap.get(replicationType);
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
LOG.debug("Getting replication pipeline forReplicationType {} :" +
" ReplicationFactor {}", replicationType.toString(),
@@ -316,7 +296,7 @@ public class PipelineSelector {
* Finalize a given pipeline.
*/
public void finalizePipeline(Pipeline pipeline) throws IOException {
- PipelineManager manager = getPipelineManager(pipeline.getType());
+ PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
if (pipeline.getLifeCycleState() == LifeCycleState.CLOSING ||
pipeline.getLifeCycleState() == LifeCycleState.CLOSED) {
@@ -327,17 +307,17 @@ public class PipelineSelector {
}
// Remove the pipeline from active allocation
- manager.finalizePipeline(pipeline);
-
- LOG.info("Finalizing pipeline. pipelineID: {}", pipeline.getId());
- updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.FINALIZE);
- closePipelineIfNoOpenContainers(pipeline);
+ if (manager.finalizePipeline(pipeline)) {
+ LOG.info("Finalizing pipeline. pipelineID: {}", pipeline.getId());
+ updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.FINALIZE);
+ closePipelineIfNoOpenContainers(pipeline);
+ }
}
/**
* Close a given pipeline.
*/
- public void closePipelineIfNoOpenContainers(Pipeline pipeline)
+ private void closePipelineIfNoOpenContainers(Pipeline pipeline)
throws IOException {
if (pipeline.getLifeCycleState() != LifeCycleState.CLOSING) {
return;
@@ -354,7 +334,7 @@ public class PipelineSelector {
* Close a given pipeline.
*/
private void closePipeline(Pipeline pipeline) throws IOException {
- PipelineManager manager = getPipelineManager(pipeline.getType());
+ PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
LOG.debug("Closing pipeline. pipelineID: {}", pipeline.getId());
HashSet<ContainerID> containers =
@@ -367,7 +347,7 @@ public class PipelineSelector {
* Add to a given pipeline.
*/
private void addOpenPipeline(Pipeline pipeline) {
- PipelineManager manager = getPipelineManager(pipeline.getType());
+ PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
LOG.debug("Adding Open pipeline. pipelineID: {}", pipeline.getId());
manager.addOpenPipeline(pipeline);
@@ -381,7 +361,7 @@ public class PipelineSelector {
}
}
- public Set<PipelineID> getPipelineId(UUID dnId) {
+ public Set<PipelineID> getPipelineByDnID(UUID dnId) {
return node2PipelineMap.getPipelines(dnId);
}
@@ -400,6 +380,9 @@ public class PipelineSelector {
pipelineMap.put(pipeline.getId(), pipeline);
pipeline2ContainerMap.put(pipeline.getId(), new HashSet<>());
node2PipelineMap.addPipeline(pipeline);
+ // reset the datanodes in the pipeline
+ // they will be reset on
+ pipeline.resetPipeline();
break;
case CLOSED:
// if the pipeline is in closed state, nothing to do.
@@ -409,6 +392,36 @@ public class PipelineSelector {
}
}
+ public void handleStaleNode(DatanodeDetails dn) {
+ Set<PipelineID> pipelineIDs = getPipelineByDnID(dn.getUuid());
+ for (PipelineID id : pipelineIDs) {
+ LOG.info("closing pipeline {}.", id);
+ eventPublisher.fireEvent(SCMEvents.PIPELINE_CLOSE, id);
+ }
+ }
+
+ void processPipelineReport(DatanodeDetails dn,
+ PipelineReportsProto pipelineReport) {
+ Set<PipelineID> reportedPipelines = new HashSet<>();
+ pipelineReport.getPipelineReportList().
+ forEach(p ->
+ reportedPipelines.add(
+ processPipelineReport(p.getPipelineID(), dn)));
+
+ //TODO: handle missing pipelines and new pipelines later
+ }
+
+ private PipelineID processPipelineReport(
+ HddsProtos.PipelineID id, DatanodeDetails dn) {
+ PipelineID pipelineID = PipelineID.getFromProtobuf(id);
+ Pipeline pipeline = pipelineMap.get(pipelineID);
+ if (pipeline != null) {
+ pipelineManagerMap.get(pipeline.getType())
+ .processPipelineReport(pipeline, dn);
+ }
+ return pipelineID;
+ }
+
/**
* Update the Pipeline State to the next state.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
index d3cec88..905a5b5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
@@ -73,20 +73,19 @@ public class RatisManagerImpl extends PipelineManager {
public Pipeline allocatePipeline(ReplicationFactor factor) {
List<DatanodeDetails> newNodesList = new LinkedList<>();
List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
- int count = getReplicationCount(factor);
//TODO: Add Raft State to the Nodes, so we can query and skip nodes from
// data from datanode instead of maintaining a set.
for (DatanodeDetails datanode : datanodes) {
Preconditions.checkNotNull(datanode);
if (!ratisMembers.contains(datanode)) {
newNodesList.add(datanode);
- if (newNodesList.size() == count) {
+ if (newNodesList.size() == factor.getNumber()) {
// once a datanode has been added to a pipeline, exclude it from
// further allocations
ratisMembers.addAll(newNodesList);
PipelineID pipelineID = PipelineID.randomId();
LOG.info("Allocating a new ratis pipeline of size: {} id: {}",
- count, pipelineID);
+ factor.getNumber(), pipelineID);
return PipelineSelector.newPipelineFromNodes(newNodesList,
ReplicationType.RATIS, factor, pipelineID);
}
@@ -103,6 +102,17 @@ public class RatisManagerImpl extends PipelineManager {
}
}
+ public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
+ super.processPipelineReport(pipeline, dn);
+ ratisMembers.add(dn);
+ }
+
+ public synchronized boolean finalizePipeline(Pipeline pipeline) {
+ activePipelines.get(pipeline.getFactor().ordinal())
+ .removePipeline(pipeline.getId());
+ return true;
+ }
+
/**
* Close the pipeline.
*/
@@ -116,29 +126,4 @@ public class RatisManagerImpl extends PipelineManager {
Preconditions.checkArgument(ratisMembers.remove(node));
}
}
-
- /**
- * list members in the pipeline .
- *
- * @param pipelineID
- * @return the datanode
- */
- @Override
- public List<DatanodeDetails> getMembers(PipelineID pipelineID)
- throws IOException {
- return null;
- }
-
- /**
- * Update the datanode list of the pipeline.
- *
- * @param pipelineID
- * @param newDatanodes
- */
- @Override
- public void updatePipeline(PipelineID pipelineID,
- List<DatanodeDetails> newDatanodes)
- throws IOException {
-
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
index ed2fc2f..045afb6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
@@ -74,18 +74,19 @@ public class StandaloneManagerImpl extends PipelineManager {
public Pipeline allocatePipeline(ReplicationFactor factor) {
List<DatanodeDetails> newNodesList = new LinkedList<>();
List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
- int count = getReplicationCount(factor);
for (DatanodeDetails datanode : datanodes) {
Preconditions.checkNotNull(datanode);
if (!standAloneMembers.contains(datanode)) {
newNodesList.add(datanode);
- if (newNodesList.size() == count) {
+ if (newNodesList.size() == factor.getNumber()) {
// once a datanode has been added to a pipeline, exclude it from
// further allocations
standAloneMembers.addAll(newNodesList);
- PipelineID pipelineID = PipelineID.randomId();
+ // Standalone pipeline use node id as pipeline
+ PipelineID pipelineID =
+ PipelineID.valueOf(newNodesList.get(0).getUuid());
LOG.info("Allocating a new standalone pipeline of size: {} id: {}",
- count, pipelineID);
+ factor.getNumber(), pipelineID);
return PipelineSelector.newPipelineFromNodes(newNodesList,
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, pipelineID);
}
@@ -98,6 +99,17 @@ public class StandaloneManagerImpl extends PipelineManager {
// Nothing to be done for standalone pipeline
}
+ public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
+ super.processPipelineReport(pipeline, dn);
+ standAloneMembers.add(dn);
+ }
+
+ public synchronized boolean finalizePipeline(Pipeline pipeline) {
+ activePipelines.get(pipeline.getFactor().ordinal())
+ .removePipeline(pipeline.getId());
+ return false;
+ }
+
/**
* Close the pipeline.
*/
@@ -107,28 +119,4 @@ public class StandaloneManagerImpl extends PipelineManager {
Preconditions.checkArgument(standAloneMembers.remove(node));
}
}
-
- /**
- * list members in the pipeline .
- *
- * @param pipelineID
- * @return the datanode
- */
- @Override
- public List<DatanodeDetails> getMembers(PipelineID pipelineID)
- throws IOException {
- return null;
- }
-
- /**
- * Update the datanode list of the pipeline.
- *
- * @param pipelineID
- * @param newDatanodes
- */
- @Override
- public void updatePipeline(PipelineID pipelineID, List<DatanodeDetails>
- newDatanodes) throws IOException {
-
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index a651f62..e65de8b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdds.scm.server;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineActionsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerActionsProto;
@@ -46,6 +48,7 @@ import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.NODE_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CMD_STATUS_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_ACTIONS;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT;
/**
* This class is responsible for dispatching heartbeat from datanode to
@@ -103,6 +106,14 @@ public final class SCMDatanodeHeartbeatDispatcher {
heartbeat.getContainerActions()));
}
+ if (heartbeat.hasPipelineReports()) {
+ LOG.debug("Dispatching Pipeline Report.");
+ eventPublisher.fireEvent(PIPELINE_REPORT,
+ new PipelineReportFromDatanode(datanodeDetails,
+ heartbeat.getPipelineReports()));
+
+ }
+
if (heartbeat.hasPipelineActions()) {
LOG.debug("Dispatching Pipeline Actions.");
eventPublisher.fireEvent(PIPELINE_ACTIONS,
@@ -179,6 +190,18 @@ public final class SCMDatanodeHeartbeatDispatcher {
}
/**
+ * Pipeline report event payload with origin.
+ */
+ public static class PipelineReportFromDatanode
+ extends ReportFromDatanode<PipelineReportsProto> {
+
+ public PipelineReportFromDatanode(DatanodeDetails datanodeDetails,
+ PipelineReportsProto report) {
+ super(datanodeDetails, report);
+ }
+ }
+
+ /**
* Pipeline action event payload with origin.
*/
public static class PipelineActionsFromDatanode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 8a09dc8..4a0d3e5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
@@ -74,7 +76,10 @@ import static org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .ReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -102,6 +107,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRES
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT;
import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
@@ -190,13 +196,14 @@ public class SCMDatanodeProtocolServer implements
public SCMRegisteredResponseProto register(
HddsProtos.DatanodeDetailsProto datanodeDetailsProto,
NodeReportProto nodeReport,
- ContainerReportsProto containerReportsProto)
+ ContainerReportsProto containerReportsProto,
+ PipelineReportsProto pipelineReportsProto)
throws IOException {
DatanodeDetails datanodeDetails = DatanodeDetails
.getFromProtoBuf(datanodeDetailsProto);
// TODO : Return the list of Nodes that forms the SCM HA.
RegisteredCommand registeredCommand = scm.getScmNodeManager()
- .register(datanodeDetails, nodeReport);
+ .register(datanodeDetails, nodeReport, pipelineReportsProto);
if (registeredCommand.getError()
== SCMRegisteredResponseProto.ErrorCode.success) {
scm.getScmContainerManager().processContainerReports(datanodeDetails,
@@ -204,6 +211,9 @@ public class SCMDatanodeProtocolServer implements
eventPublisher.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
new NodeRegistrationContainerReport(datanodeDetails,
containerReportsProto));
+ eventPublisher.fireEvent(PIPELINE_REPORT,
+ new PipelineReportFromDatanode(datanodeDetails,
+ pipelineReportsProto));
}
return getRegisteredResponse(registeredCommand);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 8e76606..2169149 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hdds.scm.node.StaleNodeHandler;
import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineCloseHandler;
import org.apache.hadoop.hdds.scm.pipelines.PipelineActionEventHandler;
+import org.apache.hadoop.hdds.scm.pipelines.PipelineReportHandler;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -217,13 +218,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new CloseContainerEventHandler(scmContainerManager);
NodeReportHandler nodeReportHandler =
new NodeReportHandler(scmNodeManager);
-
+ PipelineReportHandler pipelineReportHandler =
+ new PipelineReportHandler(
+ scmContainerManager.getPipelineSelector());
CommandStatusReportHandler cmdStatusReportHandler =
new CommandStatusReportHandler();
NewNodeHandler newNodeHandler = new NewNodeHandler(node2ContainerMap);
StaleNodeHandler staleNodeHandler =
- new StaleNodeHandler(node2ContainerMap, scmContainerManager);
+ new StaleNodeHandler(node2ContainerMap,
+ scmContainerManager.getPipelineSelector());
DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap,
getScmContainerManager().getStateManager());
ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
@@ -240,7 +244,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new PipelineActionEventHandler();
PipelineCloseHandler pipelineCloseHandler =
- new PipelineCloseHandler(scmContainerManager);
+ new PipelineCloseHandler(scmContainerManager.getPipelineSelector());
long watcherTimeout =
conf.getTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT,
@@ -300,6 +304,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
(BlockManagerImpl) scmBlockManager);
eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, clientProtocolServer);
+ eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
registerMXBean();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
index 7af9dda..24a16c7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.mockito.Mockito;
import static org.mockito.Mockito.when;
@@ -139,7 +141,8 @@ public final class TestUtils {
public static DatanodeDetails createRandomDatanodeAndRegister(
SCMNodeManager nodeManager) {
return getDatanodeDetails(
- nodeManager.register(randomDatanodeDetails(), null));
+ nodeManager.register(randomDatanodeDetails(), null,
+ getRandomPipelineReports()));
}
/**
@@ -299,6 +302,11 @@ public final class TestUtils {
return getContainerReports(containerInfos);
}
+
+ public static PipelineReportsProto getRandomPipelineReports() {
+ return PipelineReportsProto.newBuilder().build();
+ }
+
/**
* Creates container report with the given ContainerInfo(s).
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 088b700..21e44a3 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -16,6 +16,8 @@
*/
package org.apache.hadoop.hdds.scm.container;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
@@ -356,7 +358,7 @@ public class MockNodeManager implements NodeManager {
*/
@Override
public RegisteredCommand register(DatanodeDetails datanodeDetails,
- NodeReportProto nodeReport) {
+ NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index f438c8b..cbe96ee 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -286,7 +286,8 @@ public class TestNodeManager {
TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
try (SCMNodeManager nodemanager = createNodeManager(conf)) {
nodemanager.register(datanodeDetails,
- TestUtils.createNodeReport(report));
+ TestUtils.createNodeReport(report),
+ TestUtils.getRandomPipelineReports());
List<SCMCommand> command = nodemanager.processHeartbeat(datanodeDetails);
Assert.assertTrue(nodemanager.getAllNodes().contains(datanodeDetails));
Assert.assertTrue("On regular HB calls, SCM responses a "
@@ -1122,7 +1123,8 @@ public class TestNodeManager {
eq.addHandler(DATANODE_COMMAND, nodemanager);
nodemanager
- .register(datanodeDetails, TestUtils.createNodeReport(report));
+ .register(datanodeDetails, TestUtils.createNodeReport(report),
+ TestUtils.getRandomPipelineReports());
eq.fireEvent(DATANODE_COMMAND,
new CommandForDatanode<>(datanodeDetails.getUuid(),
new CloseContainerCommand(1L, ReplicationType.STAND_ALONE,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
index 14a74e9..ec1d527 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
@@ -116,7 +116,7 @@ public class TestNode2ContainerMap {
Assert.assertTrue(map.isKnownDatanode(key));
ReportResult result = map.processReport(key, values);
Assert.assertEquals(result.getStatus(),
- Node2ContainerMap.ReportStatus.ALL_IS_WELL);
+ ReportResult.ReportStatus.ALL_IS_WELL);
}
@Test
@@ -181,9 +181,9 @@ public class TestNode2ContainerMap {
UUID key = getFirstKey();
TreeSet<ContainerID> values = testData.get(key);
ReportResult result = map.processReport(key, values);
- Assert.assertEquals(Node2ContainerMap.ReportStatus.NEW_DATANODE_FOUND,
+ Assert.assertEquals(ReportResult.ReportStatus.NEW_DATANODE_FOUND,
result.getStatus());
- Assert.assertEquals(result.getNewContainers().size(), values.size());
+ Assert.assertEquals(result.getNewEntries().size(), values.size());
}
/**
@@ -216,15 +216,15 @@ public class TestNode2ContainerMap {
ReportResult result = map.processReport(key, newContainersSet);
//Assert that expected size of missing container is same as addedContainers
- Assert.assertEquals(Node2ContainerMap.ReportStatus.NEW_CONTAINERS_FOUND,
+ Assert.assertEquals(ReportResult.ReportStatus.NEW_ENTRIES_FOUND,
result.getStatus());
Assert.assertEquals(addedContainers.size(),
- result.getNewContainers().size());
+ result.getNewEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All objects are not removed.",
- result.getNewContainers().removeAll(addedContainers));
+ result.getNewEntries().removeAll(addedContainers));
}
/**
@@ -261,14 +261,14 @@ public class TestNode2ContainerMap {
//Assert that expected size of missing container is same as addedContainers
- Assert.assertEquals(Node2ContainerMap.ReportStatus.MISSING_CONTAINERS,
+ Assert.assertEquals(ReportResult.ReportStatus.MISSING_ENTRIES,
result.getStatus());
Assert.assertEquals(removedContainers.size(),
- result.getMissingContainers().size());
+ result.getMissingEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All missing containers not found.",
- result.getMissingContainers().removeAll(removedContainers));
+ result.getMissingEntries().removeAll(removedContainers));
}
@Test
@@ -307,21 +307,21 @@ public class TestNode2ContainerMap {
Assert.assertEquals(
- Node2ContainerMap.ReportStatus.MISSING_AND_NEW_CONTAINERS_FOUND,
+ ReportResult.ReportStatus.MISSING_AND_NEW_ENTRIES_FOUND,
result.getStatus());
Assert.assertEquals(removedContainers.size(),
- result.getMissingContainers().size());
+ result.getMissingEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All missing containers not found.",
- result.getMissingContainers().removeAll(removedContainers));
+ result.getMissingEntries().removeAll(removedContainers));
Assert.assertEquals(insertedSet.size(),
- result.getNewContainers().size());
+ result.getNewEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All inserted containers are not found.",
- result.getNewContainers().removeAll(insertedSet));
+ result.getNewEntries().removeAll(insertedSet));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index a513f6c..390746f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -280,7 +280,8 @@ public class TestEndPoint {
.register(nodeToRegister.getProtoBufMessage(), TestUtils
.createNodeReport(
getStorageReports(nodeToRegister.getUuid())),
- TestUtils.getRandomContainerReports(10));
+ TestUtils.getRandomContainerReports(10),
+ TestUtils.getRandomPipelineReports());
Assert.assertNotNull(responseProto);
Assert.assertEquals(nodeToRegister.getUuidString(),
responseProto.getDatanodeUUID());
@@ -308,6 +309,8 @@ public class TestEndPoint {
.createNodeReport(getStorageReports(UUID.randomUUID())));
when(ozoneContainer.getContainerReport()).thenReturn(
TestUtils.getRandomContainerReports(10));
+ when(ozoneContainer.getPipelineReport()).thenReturn(
+ TestUtils.getRandomPipelineReports());
RegisterEndpointTask endpointTask =
new RegisterEndpointTask(rpcEndPoint, conf, ozoneContainer,
mock(StateContext.class));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index a0249aa..e8a6892 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.ozone.container.testutils;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.CommandQueue;
@@ -252,7 +254,8 @@ public class ReplicationNodeManagerMock implements NodeManager {
*/
@Override
public RegisteredCommand register(DatanodeDetails dd,
- NodeReportProto nodeReport) {
+ NodeReportProto nodeReport,
+ PipelineReportsProto pipelineReportsProto) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
index aefa6b0..ad3798e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
@@ -98,7 +98,7 @@ public class TestNode2PipelineMap {
// get pipeline details by dnid
Set<PipelineID> pipelines = mapping.getPipelineSelector()
- .getPipelineId(dns.get(0).getUuid());
+ .getPipelineByDnID(dns.get(0).getUuid());
Assert.assertEquals(1, pipelines.size());
pipelines.forEach(p -> Assert.assertEquals(p,
ratisContainer.getPipeline().getId()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
index a5828e1..5eabfb9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
@@ -119,7 +119,7 @@ public class TestPipelineClose {
HddsProtos.LifeCycleState.CLOSED);
for (DatanodeDetails dn : ratisContainer1.getPipeline().getMachines()) {
// Assert that the pipeline has been removed from Node2PipelineMap as well
- Assert.assertEquals(pipelineSelector.getPipelineId(
+ Assert.assertEquals(pipelineSelector.getPipelineByDnID(
dn.getUuid()).size(), 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
index 3999d76..fb94b3c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.ContainerMapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -87,7 +88,7 @@ public class TestSCMRestart {
}
@Test
- public void testPipelineWithScmRestart() {
+ public void testPipelineWithScmRestart() throws IOException {
// After restart make sure that the pipeline are still present
Pipeline ratisPipeline1AfterRestart = newMapping.getPipelineSelector()
.getPipeline(ratisPipeline1.getId());
@@ -97,5 +98,22 @@ public class TestSCMRestart {
Assert.assertNotSame(ratisPipeline2AfterRestart, ratisPipeline2);
Assert.assertEquals(ratisPipeline1AfterRestart, ratisPipeline1);
Assert.assertEquals(ratisPipeline2AfterRestart, ratisPipeline2);
+
+ for (DatanodeDetails dn : ratisPipeline1.getMachines()) {
+ Assert.assertEquals(dn, ratisPipeline1AfterRestart.getDatanodes()
+ .get(dn.getUuidString()));
+ }
+
+ for (DatanodeDetails dn : ratisPipeline2.getMachines()) {
+ Assert.assertEquals(dn, ratisPipeline2AfterRestart.getDatanodes()
+ .get(dn.getUuidString()));
+ }
+
+ // Try creating a new ratis pipeline, it should be from the same pipeline
+ // as was before restart
+ Pipeline newRatisPipeline =
+ newMapping.allocateContainer(RATIS, THREE, "Owner1")
+ .getPipeline();
+ Assert.assertEquals(newRatisPipeline.getId(), ratisPipeline1.getId());
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
index 1cb2cda..a83c16e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
@@ -36,8 +36,12 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+
/**
* Helpers for Ratis tests.
*/
@@ -60,6 +64,7 @@ public interface RatisTestHelper {
public RatisTestSuite()
throws IOException, TimeoutException, InterruptedException {
conf = newOzoneConfiguration(RPC);
+
cluster = newMiniOzoneCluster(NUM_DATANODES, conf);
}
@@ -96,6 +101,8 @@ public interface RatisTestHelper {
static void initRatisConf(RpcType rpc, Configuration conf) {
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY, rpc.name());
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS);
LOG.info(OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY
+ " = " + rpc.name());
}
@@ -104,6 +111,8 @@ public interface RatisTestHelper {
int numDatanodes, OzoneConfiguration conf)
throws IOException, TimeoutException, InterruptedException {
final MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
+ .setHbInterval(1000)
+ .setHbProcessorInterval(1000)
.setNumDatanodes(numDatanodes).build();
cluster.waitForClusterToBeReady();
return cluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 6377f11..02cd985 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -136,6 +136,7 @@ public class TestKeys {
ozoneCluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1)
.setHbInterval(1000)
+ .setHbProcessorInterval(1000)
.build();
ozoneCluster.waitForClusterToBeReady();
client = new RpcClient(conf);
@@ -328,7 +329,6 @@ public class TestKeys {
cluster.restartHddsDatanode(datanodeIdx);
}
- @Ignore("Causes a JVm exit")
@Test
public void testPutAndGetKeyWithDnRestart() throws Exception {
runTestPutAndGetKeyWithDnRestart(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
index 915d0f6..2e8f539 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
@@ -26,7 +26,6 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.Ignore;
import org.junit.rules.Timeout;
import static org.apache.hadoop.ozone.web.client
@@ -83,7 +82,6 @@ public class TestKeysRatis {
getMultiPartKey(delimiter)));
}
- @Ignore("disabling for now, datanodes restart with ratis is buggy")
@Test
public void testPutAndGetKeyWithDnRestart() throws Exception {
runTestPutAndGetKeyWithDnRestart(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 275ae6e..caf6d4f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -101,7 +101,7 @@
<ldap-api.version>1.0.0-M33</ldap-api.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-50588bd-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-eca3531-SNAPSHOT</ratis.version>
<jcache.version>1.0-alpha-1</jcache.version>
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDDS-468. Add version number to
datanode plugin and ozone file system jar. Contributed by Bharat Viswanadham.
Posted by xk...@apache.org.
HDDS-468. Add version number to datanode plugin and ozone file system jar. Contributed by Bharat Viswanadham.
(cherry picked from commit a71aea732315ffc5c99c011369adfbeddb9a8c01)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6ad84eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6ad84eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6ad84eb
Branch: refs/heads/HDFS-12943
Commit: b6ad84ebb1bf5f5f4c1fffe55a6b529ee43dd7a7
Parents: 001611c
Author: Márton Elek <el...@apache.org>
Authored: Tue Sep 18 11:00:51 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Tue Sep 18 11:01:52 2018 +0200
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 4 ++--
hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml | 2 +-
hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ad84eb/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
index f047c87..00854b4 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -145,11 +145,11 @@ run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}"
#shaded ozonefs
mkdir -p "./share/hadoop/ozonefs"
-cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
+cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
#shaded datanode service
mkdir -p "./share/hadoop/ozoneplugin"
-cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin.jar"
+cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar"
# Optional documentation, could be missing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ad84eb/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
index ed18e5c..b89052d 100644
--- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
@@ -33,7 +33,7 @@ services:
- ../..:/opt/ozone
command: ["hdfs","datanode"]
environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin.jar
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/*.jar
env_file:
- ./docker-config
ozoneManager:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ad84eb/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
index 1671c39..a1e8748 100644
--- a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
@@ -55,5 +55,5 @@ services:
env_file:
- ./docker-config
environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar
command: ["watch","-n","100000","ls"]
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDDS-495. Ozone docs and ozonefs
packages have undefined hadoop component. Contributed by Dinesh Chitlangia.
Posted by xk...@apache.org.
HDDS-495. Ozone docs and ozonefs packages have undefined hadoop component. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58963727
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58963727
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58963727
Branch: refs/heads/HDFS-12943
Commit: 589637276105b0f9b9d5b7f6207f6ad0892f0b28
Parents: 2df0a8d
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Tue Sep 18 13:50:28 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Tue Sep 18 13:50:39 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/pom.xml | 5 +++++
hadoop-ozone/ozonefs/pom.xml | 2 ++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58963727/hadoop-ozone/docs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/pom.xml b/hadoop-ozone/docs/pom.xml
index d8edd15..64d0ec8 100644
--- a/hadoop-ozone/docs/pom.xml
+++ b/hadoop-ozone/docs/pom.xml
@@ -28,6 +28,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Documentation</name>
<packaging>jar</packaging>
+ <properties>
+ <hadoop.component>ozone</hadoop.component>
+ <is.hadoop.component>true</is.hadoop.component>
+ </properties>
+
<dependencies>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58963727/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index 8174b74..c45aacd 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -28,6 +28,8 @@
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
+ <hadoop.component>ozone</hadoop.component>
+ <is.hadoop.component>true</is.hadoop.component>
</properties>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDDS-501.
AllocateBlockResponse.keyLocation must be an optional field. Contributed by
Arpit Agarwal.
Posted by xk...@apache.org.
HDDS-501. AllocateBlockResponse.keyLocation must be an optional field. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f176e8a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f176e8a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f176e8a3
Branch: refs/heads/HDFS-12943
Commit: f176e8a3aeca2f72896a55e9d28d320ce3d3f76c
Parents: 44857476
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 15:41:28 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 15:41:28 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f176e8a3/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 242e3b5..975c790 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -324,7 +324,7 @@ message AllocateBlockRequest {
message AllocateBlockResponse {
required Status status = 1;
- required KeyLocation keyLocation = 2;
+ optional KeyLocation keyLocation = 2;
}
message CommitKeyRequest {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-8757. [Submarine] Add
Tensorboard component when --tensorboard is specified. Contributed by Wangda
Tan.
Posted by xk...@apache.org.
YARN-8757. [Submarine] Add Tensorboard component when --tensorboard is specified. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1824d5d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1824d5d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1824d5d1
Branch: refs/heads/HDFS-12943
Commit: 1824d5d1c49c16db6341141fa204d4a4c02d0944
Parents: 56e0d63
Author: Sunil G <su...@apache.org>
Authored: Wed Sep 19 22:18:55 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Sep 19 22:18:55 2018 +0530
----------------------------------------------------------------------
.../yarn/submarine/client/cli/CliConstants.java | 6 +
.../yarn/submarine/client/cli/CliUtils.java | 10 +-
.../yarn/submarine/client/cli/RunJobCli.java | 43 ++-
.../client/cli/param/RunJobParameters.java | 52 +++-
.../fs/DefaultRemoteDirectoryManager.java | 21 +-
.../common/fs/RemoteDirectoryManager.java | 4 +-
.../common/FSBasedSubmarineStorageImpl.java | 4 +-
.../yarnservice/YarnServiceJobSubmitter.java | 126 ++++++---
.../runtimes/yarnservice/YarnServiceUtils.java | 26 +-
.../yarnservice/TestYarnServiceRunJobCli.java | 268 ++++++++++++++++---
.../common/fs/MockRemoteDirectoryManager.java | 7 +-
11 files changed, 463 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
index d0958a8..d51ffc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
@@ -35,6 +35,10 @@ public class CliConstants {
public static final String DOCKER_IMAGE = "docker_image";
public static final String QUEUE = "queue";
public static final String TENSORBOARD = "tensorboard";
+ public static final String TENSORBOARD_RESOURCES = "tensorboard_resources";
+ public static final String TENSORBOARD_DEFAULT_RESOURCES =
+ "memory=4G,vcores=1";
+
public static final String WORKER_LAUNCH_CMD = "worker_launch_cmd";
public static final String SERVING_LAUNCH_CMD = "serving_launch_cmd";
public static final String PS_LAUNCH_CMD = "ps_launch_cmd";
@@ -45,4 +49,6 @@ public class CliConstants {
public static final String WAIT_JOB_FINISH = "wait_job_finish";
public static final String PS_DOCKER_IMAGE = "ps_docker_image";
public static final String WORKER_DOCKER_IMAGE = "worker_docker_image";
+ public static final String TENSORBOARD_DOCKER_IMAGE =
+ "tensorboard_docker_image";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
index 6dd3e4d..546c6eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
@@ -39,17 +39,9 @@ public class CliUtils {
public static String replacePatternsInLaunchCommand(String specifiedCli,
RunJobParameters jobRunParameters,
RemoteDirectoryManager directoryManager) throws IOException {
- String jobDir = jobRunParameters.getCheckpointPath();
- if (null == jobDir) {
- jobDir = directoryManager.getJobCheckpointDir(jobRunParameters.getName(),
- true).toString();
- }
-
String input = jobRunParameters.getInputPath();
+ String jobDir = jobRunParameters.getCheckpointPath();
String savedModelDir = jobRunParameters.getSavedModelPath();
- if (null == savedModelDir) {
- savedModelDir = jobDir;
- }
Map<String, String> replacePattern = new HashMap<>();
if (jobDir != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
index d7dfc0d..faa22d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
@@ -89,8 +89,16 @@ public class RunJobCli extends AbstractCli {
options.addOption(CliConstants.DOCKER_IMAGE, true, "Docker image name/tag");
options.addOption(CliConstants.QUEUE, true,
"Name of queue to run the job, by default it uses default queue");
- options.addOption(CliConstants.TENSORBOARD, true,
- "Should we run TensorBoard" + " for this job? By default it's true");
+ options.addOption(CliConstants.TENSORBOARD, false,
+ "Should we run TensorBoard"
+ + " for this job? By default it's disabled");
+ options.addOption(CliConstants.TENSORBOARD_RESOURCES, true,
+ "Specify resources of Tensorboard, by default it is "
+ + CliConstants.TENSORBOARD_DEFAULT_RESOURCES);
+ options.addOption(CliConstants.TENSORBOARD_DOCKER_IMAGE, true,
+ "Specify Tensorboard docker image. when this is not "
+ + "specified, Tensorboard " + "uses --" + CliConstants.DOCKER_IMAGE
+ + " as default.");
options.addOption(CliConstants.WORKER_LAUNCH_CMD, true,
"Commandline of worker, arguments will be "
+ "directly used to launch the worker");
@@ -144,10 +152,39 @@ public class RunJobCli extends AbstractCli {
throw e;
}
+ // Set default job dir / saved model dir, etc.
+ setDefaultDirs();
+
// replace patterns
replacePatternsInParameters();
}
+ private void setDefaultDirs() throws IOException {
+ // Create directories if needed
+ String jobDir = parameters.getCheckpointPath();
+ if (null == jobDir) {
+ if (parameters.getNumWorkers() > 0) {
+ jobDir = clientContext.getRemoteDirectoryManager().getJobCheckpointDir(
+ parameters.getName(), true).toString();
+ } else {
+ // when #workers == 0, it means we only launch TB. In that case,
+ // point job dir to root dir so all job's metrics will be shown.
+ jobDir = clientContext.getRemoteDirectoryManager().getUserRootFolder()
+ .toString();
+ }
+ parameters.setCheckpointPath(jobDir);
+ }
+
+ if (parameters.getNumWorkers() > 0) {
+ // Only do this when #worker > 0
+ String savedModelDir = parameters.getSavedModelPath();
+ if (null == savedModelDir) {
+ savedModelDir = jobDir;
+ parameters.setSavedModelPath(savedModelDir);
+ }
+ }
+ }
+
private void storeJobInformation(String jobName, ApplicationId applicationId,
String[] args) throws IOException {
Map<String, String> jobInfo = new HashMap<>();
@@ -198,7 +235,7 @@ public class RunJobCli extends AbstractCli {
}
@VisibleForTesting
- RunJobParameters getRunJobParameters() {
+ public RunJobParameters getRunJobParameters() {
return parameters;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
index 6cab9e3..4558f6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
@@ -37,6 +37,8 @@ public class RunJobParameters extends RunParameters {
private Resource workerResource;
private Resource psResource;
private boolean tensorboardEnabled;
+ private Resource tensorboardResource;
+ private String tensorboardDockerImage;
private String workerLaunchCmd;
private String psLaunchCmd;
@@ -69,19 +71,23 @@ public class RunJobParameters extends RunParameters {
// When distributed training is required
if (nWorkers >= 2 && nPS > 0) {
distributed = true;
- } else if (nWorkers == 1 && nPS > 0) {
+ } else if (nWorkers <= 1 && nPS > 0) {
throw new ParseException("Only specified one worker but non-zero PS, "
+ "please double check.");
}
- String workerResourceStr = parsedCommandLine.getOptionValue(
- CliConstants.WORKER_RES);
- if (workerResourceStr == null) {
- throw new ParseException("--" + CliConstants.WORKER_RES + " is absent.");
+ workerResource = null;
+ if (nWorkers > 0) {
+ String workerResourceStr = parsedCommandLine.getOptionValue(
+ CliConstants.WORKER_RES);
+ if (workerResourceStr == null) {
+ throw new ParseException(
+ "--" + CliConstants.WORKER_RES + " is absent.");
+ }
+ workerResource = CliUtils.createResourceFromString(
+ workerResourceStr,
+ clientContext.getOrCreateYarnClient().getResourceTypeInfo());
}
- Resource workerResource = CliUtils.createResourceFromString(
- workerResourceStr,
- clientContext.getOrCreateYarnClient().getResourceTypeInfo());
Resource psResource = null;
if (nPS > 0) {
@@ -94,9 +100,19 @@ public class RunJobParameters extends RunParameters {
}
boolean tensorboard = false;
- if (parsedCommandLine.getOptionValue(CliConstants.TENSORBOARD) != null) {
- tensorboard = Boolean.parseBoolean(
- parsedCommandLine.getOptionValue(CliConstants.TENSORBOARD));
+ if (parsedCommandLine.hasOption(CliConstants.TENSORBOARD)) {
+ tensorboard = true;
+ String tensorboardResourceStr = parsedCommandLine.getOptionValue(
+ CliConstants.TENSORBOARD_RESOURCES);
+ if (tensorboardResourceStr == null || tensorboardResourceStr.isEmpty()) {
+ tensorboardResourceStr = CliConstants.TENSORBOARD_DEFAULT_RESOURCES;
+ }
+ tensorboardResource = CliUtils.createResourceFromString(
+ tensorboardResourceStr,
+ clientContext.getOrCreateYarnClient().getResourceTypeInfo());
+ tensorboardDockerImage = parsedCommandLine.getOptionValue(
+ CliConstants.TENSORBOARD_DOCKER_IMAGE);
+ this.setTensorboardResource(tensorboardResource);
}
if (parsedCommandLine.hasOption(CliConstants.WAIT_JOB_FINISH)) {
@@ -115,7 +131,7 @@ public class RunJobParameters extends RunParameters {
this.setInputPath(input).setCheckpointPath(jobDir).setNumPS(nPS).setNumWorkers(nWorkers)
.setPSLaunchCmd(psLaunchCommand).setWorkerLaunchCmd(workerLaunchCmd)
- .setPsResource(psResource).setWorkerResource(workerResource)
+ .setPsResource(psResource)
.setTensorboardEnabled(tensorboard);
super.updateParametersByParsedCommandline(parsedCommandLine,
@@ -219,4 +235,16 @@ public class RunJobParameters extends RunParameters {
public boolean isDistributed() {
return distributed;
}
+
+ public Resource getTensorboardResource() {
+ return tensorboardResource;
+ }
+
+ public void setTensorboardResource(Resource tensorboardResource) {
+ this.tensorboardResource = tensorboardResource;
+ }
+
+ public String getTensorboardDockerImage() {
+ return tensorboardDockerImage;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
index fe8956a..b2e2b41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
@@ -14,6 +14,7 @@
package org.apache.hadoop.yarn.submarine.common.fs;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.submarine.client.cli.CliConstants;
@@ -42,7 +43,10 @@ public class DefaultRemoteDirectoryManager implements RemoteDirectoryManager {
if (create) {
createFolderIfNotExist(staging);
}
- return staging;
+
+ // Get a file status to make sure it is a absolute path.
+ FileStatus fStatus = fs.getFileStatus(staging);
+ return fStatus.getPath();
}
@Override
@@ -70,8 +74,21 @@ public class DefaultRemoteDirectoryManager implements RemoteDirectoryManager {
return fs;
}
+ @Override
+ public Path getUserRootFolder() throws IOException {
+ Path rootPath = new Path("submarine", "jobs");
+ createFolderIfNotExist(rootPath);
+ // Get a file status to make sure it is a absolute path.
+ FileStatus fStatus = fs.getFileStatus(rootPath);
+ return fStatus.getPath();
+ }
+
private Path getJobRootFolder(String jobName) throws IOException {
- return new Path(new Path("submarine", "jobs"), jobName);
+ Path jobRootPath = getUserRootFolder();
+ createFolderIfNotExist(jobRootPath);
+ // Get a file status to make sure it is a absolute path.
+ FileStatus fStatus = fs.getFileStatus(jobRootPath);
+ return fStatus.getPath();
}
private void createFolderIfNotExist(Path path) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
index 132b314..ad0d428 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
@@ -27,4 +27,6 @@ public interface RemoteDirectoryManager {
Path getModelDir(String modelName, boolean create) throws IOException;
FileSystem getFileSystem() throws IOException;
-}
+
+ Path getUserRootFolder() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
index ebf9581..767fe78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
@@ -32,11 +32,9 @@ import java.util.Map;
* A super naive FS-based storage.
*/
public class FSBasedSubmarineStorageImpl extends SubmarineStorage {
- ClientContext clientContext;
RemoteDirectoryManager rdm;
public FSBasedSubmarineStorageImpl(ClientContext clientContext) {
- this.clientContext = clientContext;
rdm = clientContext.getRemoteDirectoryManager();
}
@@ -89,7 +87,7 @@ public class FSBasedSubmarineStorageImpl extends SubmarineStorage {
private Map<String, String> deserializeMap(FSDataInputStream fis)
throws IOException {
ObjectInput oi = new ObjectInputStream(fis);
- Map<String, String> newMap = null;
+ Map<String, String> newMap;
try {
newMap = (Map<String, String>) oi.readObject();
} catch (ClassNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index a2a2067..8fb213f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -15,6 +15,7 @@
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -59,6 +60,10 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
Service serviceSpec;
private Set<Path> uploadedFiles = new HashSet<>();
+ // Used by testing
+ private Map<String, String> componentToLocalLaunchScriptPath =
+ new HashMap<>();
+
public YarnServiceJobSubmitter(ClientContext clientContext) {
this.clientContext = clientContext;
}
@@ -186,6 +191,14 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
envs.put(Envs.TASK_TYPE_ENV, taskType.name());
}
+ private String getUserName() {
+ return System.getProperty("user.name");
+ }
+
+ private String getDNSDomain() {
+ return clientContext.getYarnConfig().get("hadoop.registry.dns.domain-name");
+ }
+
/*
* Generate a command launch script on local disk, returns patch to the script
*/
@@ -194,50 +207,48 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
FileWriter fw = new FileWriter(file);
- fw.append("#!/bin/bash\n");
-
- addHdfsClassPathIfNeeded(parameters, fw, comp);
-
- // For primary_worker
- if (taskType == TaskType.PRIMARY_WORKER) {
- // Do we need tensorboard?
- if (parameters.isTensorboardEnabled()) {
- int tensorboardPort = 6006;
- // Run tensorboard at the background
- fw.append(
- "tensorboard --port " + tensorboardPort + " --logdir " + parameters
- .getCheckpointPath() + " &\n");
- }
- }
-
- // When distributed training is required
- if (parameters.isDistributed()) {
- // Generated TF_CONFIG
- String tfConfigEnv = YarnServiceUtils.getTFConfigEnv(
- taskType.getComponentName(), parameters.getNumWorkers(),
- parameters.getNumPS(), parameters.getName(),
- System.getProperty("user.name"),
- clientContext.getYarnConfig().get("hadoop.registry.dns.domain-name"));
- fw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
- }
+ try {
+ fw.append("#!/bin/bash\n");
- // Print launch command
- if (taskType.equals(TaskType.WORKER) || taskType.equals(
- TaskType.PRIMARY_WORKER)) {
- fw.append(parameters.getWorkerLaunchCmd() + '\n');
+ addHdfsClassPathIfNeeded(parameters, fw, comp);
- if (SubmarineLogs.isVerbose()) {
- LOG.info("Worker command =[" + parameters.getWorkerLaunchCmd() + "]");
- }
- } else if (taskType.equals(TaskType.PS)) {
- fw.append(parameters.getPSLaunchCmd() + '\n');
+ if (taskType.equals(TaskType.TENSORBOARD)) {
+ String tbCommand =
+ "export LC_ALL=C && tensorboard --logdir=" + parameters
+ .getCheckpointPath();
+ fw.append(tbCommand + "\n");
+ LOG.info("Tensorboard command=" + tbCommand);
+ } else{
+ // When distributed training is required
+ if (parameters.isDistributed()) {
+ // Generated TF_CONFIG
+ String tfConfigEnv = YarnServiceUtils.getTFConfigEnv(
+ taskType.getComponentName(), parameters.getNumWorkers(),
+ parameters.getNumPS(), parameters.getName(), getUserName(),
+ getDNSDomain());
+ fw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
+ }
- if (SubmarineLogs.isVerbose()) {
- LOG.info("PS command =[" + parameters.getPSLaunchCmd() + "]");
+ // Print launch command
+ if (taskType.equals(TaskType.WORKER) || taskType.equals(
+ TaskType.PRIMARY_WORKER)) {
+ fw.append(parameters.getWorkerLaunchCmd() + '\n');
+
+ if (SubmarineLogs.isVerbose()) {
+ LOG.info(
+ "Worker command =[" + parameters.getWorkerLaunchCmd() + "]");
+ }
+ } else if (taskType.equals(TaskType.PS)) {
+ fw.append(parameters.getPSLaunchCmd() + '\n');
+
+ if (SubmarineLogs.isVerbose()) {
+ LOG.info("PS command =[" + parameters.getPSLaunchCmd() + "]");
+ }
+ }
}
+ } finally {
+ fw.close();
}
-
- fw.close();
return file.getAbsolutePath();
}
@@ -320,6 +331,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
destScriptFileName, component);
component.setLaunchCommand("./" + destScriptFileName);
+ componentToLocalLaunchScriptPath.put(taskType.getComponentName(),
+ localScriptFile);
}
private void addWorkerComponent(Service service,
@@ -410,6 +423,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
private Service createServiceByParameters(RunJobParameters parameters)
throws IOException {
+ componentToLocalLaunchScriptPath.clear();
Service service = new Service();
service.setName(parameters.getName());
service.setVersion(String.valueOf(System.currentTimeMillis()));
@@ -417,7 +431,9 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
handleServiceEnvs(service, parameters);
- addWorkerComponents(service, parameters);
+ if (parameters.getNumWorkers() > 0) {
+ addWorkerComponents(service, parameters);
+ }
if (parameters.getNumPS() > 0) {
Component psComponent = new Component();
@@ -436,6 +452,31 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
handleLaunchCommand(parameters, TaskType.PS, psComponent);
service.addComponent(psComponent);
}
+
+ if (parameters.isTensorboardEnabled()) {
+ Component tbComponent = new Component();
+ tbComponent.setName(TaskType.TENSORBOARD.getComponentName());
+ addCommonEnvironments(tbComponent, TaskType.TENSORBOARD);
+ tbComponent.setNumberOfContainers(1L);
+ tbComponent.setRestartPolicy(Component.RestartPolicyEnum.NEVER);
+ tbComponent.setResource(getServiceResourceFromYarnResource(
+ parameters.getTensorboardResource()));
+ if (parameters.getTensorboardDockerImage() != null) {
+ tbComponent.setArtifact(
+ getDockerArtifact(parameters.getTensorboardDockerImage()));
+ }
+
+ handleLaunchCommand(parameters, TaskType.TENSORBOARD, tbComponent);
+
+ // Add tensorboard to quicklink
+ String tensorboardLink = "http://" + YarnServiceUtils.getDNSName(
+ parameters.getName(), TaskType.TENSORBOARD.getComponentName(), 0,
+ getUserName(), getDNSDomain(), 6006);
+ LOG.info("Link to tensorboard:" + tensorboardLink);
+ service.addComponent(tbComponent);
+ service.setQuicklinks(ImmutableMap.of("Tensorboard", tensorboardLink));
+ }
+
return service;
}
@@ -458,4 +499,9 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
public Service getServiceSpec() {
return serviceSpec;
}
+
+ @VisibleForTesting
+ public Map<String, String> getComponentToLocalLaunchScriptPath() {
+ return componentToLocalLaunchScriptPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
index f7ecc97..9238a67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
@@ -40,10 +40,23 @@ public class YarnServiceUtils {
YarnServiceUtils.stubServiceClient = stubServiceClient;
}
+ public static String getDNSName(String serviceName, String componentName,
+ int index, String userName, String domain, int port) {
+ return componentName + "-" + index + getDNSNameCommonSuffix(serviceName,
+ userName, domain, port);
+ }
+
+ private static String getDNSNameCommonSuffix(String serviceName,
+ String userName, String domain, int port) {
+ String commonEndpointSuffix =
+ "." + serviceName + "." + userName + "." + domain + ":" + port;
+ return commonEndpointSuffix;
+ }
+
public static String getTFConfigEnv(String curCommponentName, int nWorkers,
int nPs, String serviceName, String userName, String domain) {
- String commonEndpointSuffix =
- "." + serviceName + "." + userName + "." + domain + ":8000";
+ String commonEndpointSuffix = getDNSNameCommonSuffix(serviceName, userName,
+ domain, 8000);
String json = "{\\\"cluster\\\":{";
@@ -58,7 +71,14 @@ public class YarnServiceUtils {
+ " \\\"index\\\":" + '$' + Envs.TASK_INDEX_ENV + "},";
String environment = "\\\"environment\\\":\\\"cloud\\\"}";
- return json + master + worker + ps + task + environment;
+ StringBuilder sb = new StringBuilder();
+ sb.append(json);
+ sb.append(master);
+ sb.append(worker);
+ sb.append(ps);
+ sb.append(task);
+ sb.append(environment);
+ return sb.toString();
}
private static String getComponentArrayJson(String componentName, int count,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
index e1756b8..a88d673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.submarine.client.cli.yarnservice;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
@@ -32,11 +33,15 @@ import org.apache.hadoop.yarn.submarine.runtimes.common.StorageKeyConstants;
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceJobSubmitter;
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Map;
import static org.mockito.Matchers.any;
@@ -65,25 +70,8 @@ public class TestYarnServiceRunJobCli {
return ((YarnServiceJobSubmitter) jobSubmitter).getServiceSpec();
}
- @Test
- public void testBasicRunJobForDistributedTraining() throws Exception {
- MockClientContext mockClientContext =
- YarnServiceCliTestUtils.getMockClientContext();
- RunJobCli runJobCli = new RunJobCli(mockClientContext);
- Assert.assertFalse(SubmarineLogs.isVerbose());
-
- runJobCli.run(
- new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
- "--input_path", "s3://input", "--checkpoint_path",
- "s3://output", "--num_workers", "3", "--num_ps", "2",
- "--worker_launch_cmd", "python run-job.py", "--worker_resources",
- "memory=2048M,vcores=2", "--ps_resources", "memory=4096M,vcores=4",
- "--tensorboard", "true", "--ps_docker_image", "ps.image",
- "--worker_docker_image", "worker.image",
- "--ps_launch_cmd", "python run-ps.py", "--verbose" });
- Service serviceSpec = getServiceSpecFromJobSubmitter(
- runJobCli.getJobSubmitter());
- Assert.assertEquals(3, serviceSpec.getComponents().size());
+ private void commonVerifyDistributedTrainingSpec(Service serviceSpec)
+ throws Exception {
Assert.assertTrue(
serviceSpec.getComponent(TaskType.WORKER.getComponentName()) != null);
Assert.assertTrue(
@@ -98,7 +86,7 @@ public class TestYarnServiceRunJobCli {
primaryWorkerComp.getResource().getCpus().intValue());
Component workerComp = serviceSpec.getComponent(
- TaskType.WORKER.getComponentName());
+ TaskType.WORKER.getComponentName());
Assert.assertEquals(2048, workerComp.getResource().calcMemoryMB());
Assert.assertEquals(2, workerComp.getResource().getCpus().intValue());
@@ -110,8 +98,55 @@ public class TestYarnServiceRunJobCli {
Assert.assertEquals("ps.image", psComp.getArtifact().getId());
Assert.assertTrue(SubmarineLogs.isVerbose());
+ }
+
+ @Test
+ public void testBasicRunJobForDistributedTraining() throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
- // TODO, ADD TEST TO USE SERVICE CLIENT TO VALIDATE THE JSON SPEC
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+ "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+ "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+ "ps.image", "--worker_docker_image", "worker.image",
+ "--ps_launch_cmd", "python run-ps.py", "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(3, serviceSpec.getComponents().size());
+
+ commonVerifyDistributedTrainingSpec(serviceSpec);
+ }
+
+ @Test
+ public void testBasicRunJobForDistributedTrainingWithTensorboard()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+ "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+ "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+ "ps.image", "--worker_docker_image", "worker.image",
+ "--tensorboard", "--ps_launch_cmd", "python run-ps.py",
+ "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(4, serviceSpec.getComponents().size());
+
+ commonVerifyDistributedTrainingSpec(serviceSpec);
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
}
@Test
@@ -123,13 +158,84 @@ public class TestYarnServiceRunJobCli {
runJobCli.run(
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
- "--input_path", "s3://input", "--checkpoint_path",
- "s3://output", "--num_workers", "1", "--worker_launch_cmd",
- "python run-job.py", "--worker_resources", "memory=2G,vcores=2",
- "--tensorboard", "true", "--verbose" });
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=2G,vcores=2", "--verbose" });
+
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ commonTestSingleNodeTraining(serviceSpec);
+ }
+
+ @Test
+ public void testTensorboardOnlyService() throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "0", "--tensorboard", "--verbose" });
+
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
+ }
+
+ @Test
+ public void testTensorboardOnlyServiceWithCustomizedDockerImageAndResourceCkptPath()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "0", "--tensorboard", "--verbose",
+ "--tensorboard_resources", "memory=2G,vcores=2",
+ "--tensorboard_docker_image", "tb_docker_image:001" });
+
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(2048, 2));
+ }
+
+ @Test
+ public void testTensorboardOnlyServiceWithCustomizedDockerImageAndResource()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--num_workers", "0", "--tensorboard", "--verbose",
+ "--tensorboard_resources", "memory=2G,vcores=2",
+ "--tensorboard_docker_image", "tb_docker_image:001" });
+
Service serviceSpec = getServiceSpecFromJobSubmitter(
runJobCli.getJobSubmitter());
Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(2048, 2));
+ }
+
+ private void commonTestSingleNodeTraining(Service serviceSpec)
+ throws Exception {
Assert.assertTrue(
serviceSpec.getComponent(TaskType.PRIMARY_WORKER.getComponentName())
!= null);
@@ -140,8 +246,110 @@ public class TestYarnServiceRunJobCli {
primaryWorkerComp.getResource().getCpus().intValue());
Assert.assertTrue(SubmarineLogs.isVerbose());
+ }
+
+ private void verifyTensorboardComponent(RunJobCli runJobCli,
+ Service serviceSpec, Resource resource) throws Exception {
+ Assert.assertTrue(
+ serviceSpec.getComponent(TaskType.TENSORBOARD.getComponentName())
+ != null);
+ Component tensorboardComp = serviceSpec.getComponent(
+ TaskType.TENSORBOARD.getComponentName());
+ Assert.assertEquals(1, tensorboardComp.getNumberOfContainers().intValue());
+ Assert.assertEquals(resource.getMemorySize(),
+ tensorboardComp.getResource().calcMemoryMB());
+ Assert.assertEquals(resource.getVirtualCores(),
+ tensorboardComp.getResource().getCpus().intValue());
+
+ Assert.assertEquals("./run-TENSORBOARD.sh",
+ tensorboardComp.getLaunchCommand());
+
+ // Check docker image
+ if (runJobCli.getRunJobParameters().getTensorboardDockerImage() != null) {
+ Assert.assertEquals(
+ runJobCli.getRunJobParameters().getTensorboardDockerImage(),
+ tensorboardComp.getArtifact().getId());
+ } else{
+ Assert.assertNull(tensorboardComp.getArtifact());
+ }
+
+ YarnServiceJobSubmitter yarnServiceJobSubmitter =
+ (YarnServiceJobSubmitter) runJobCli.getJobSubmitter();
+
+ String expectedLaunchScript =
+ "#!/bin/bash\n" + "echo \"CLASSPATH:$CLASSPATH\"\n"
+ + "echo \"HADOOP_CONF_DIR:$HADOOP_CONF_DIR\"\n"
+ + "echo \"HADOOP_TOKEN_FILE_LOCATION:$HADOOP_TOKEN_FILE_LOCATION\"\n"
+ + "echo \"JAVA_HOME:$JAVA_HOME\"\n"
+ + "echo \"LD_LIBRARY_PATH:$LD_LIBRARY_PATH\"\n"
+ + "echo \"HADOOP_HDFS_HOME:$HADOOP_HDFS_HOME\"\n"
+ + "export LC_ALL=C && tensorboard --logdir=" + runJobCli
+ .getRunJobParameters().getCheckpointPath() + "\n";
+
+ verifyLaunchScriptForComponet(yarnServiceJobSubmitter, serviceSpec,
+ TaskType.TENSORBOARD, expectedLaunchScript);
+ }
+
+ private void verifyLaunchScriptForComponet(
+ YarnServiceJobSubmitter yarnServiceJobSubmitter, Service serviceSpec,
+ TaskType taskType, String expectedLaunchScriptContent) throws Exception {
+ Map<String, String> componentToLocalLaunchScriptMap =
+ yarnServiceJobSubmitter.getComponentToLocalLaunchScriptPath();
+
+ String path = componentToLocalLaunchScriptMap.get(
+ taskType.getComponentName());
+
+ byte[] encoded = Files.readAllBytes(Paths.get(path));
+ String scriptContent = new String(encoded, Charset.defaultCharset());
+
+ Assert.assertEquals(expectedLaunchScriptContent, scriptContent);
+ }
+
+ @Test
+ public void testBasicRunJobForSingleNodeTrainingWithTensorboard()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=2G,vcores=2", "--tensorboard",
+ "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+
+ Assert.assertEquals(2, serviceSpec.getComponents().size());
+
+ commonTestSingleNodeTraining(serviceSpec);
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
+ }
+
+ @Test
+ public void testBasicRunJobForSingleNodeTrainingWithGeneratedCheckpoint()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--num_workers", "1",
+ "--worker_launch_cmd", "python run-job.py", "--worker_resources",
+ "memory=2G,vcores=2", "--tensorboard", "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+
+ Assert.assertEquals(2, serviceSpec.getComponents().size());
- // TODO, ADD TEST TO USE SERVICE CLIENT TO VALIDATE THE JSON SPEC
+ commonTestSingleNodeTraining(serviceSpec);
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
}
@Test
@@ -153,10 +361,10 @@ public class TestYarnServiceRunJobCli {
runJobCli.run(
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
- "--input_path", "s3://input", "--checkpoint_path",
- "s3://output", "--num_workers", "1", "--worker_launch_cmd",
- "python run-job.py", "--worker_resources", "memory=2G,vcores=2",
- "--tensorboard", "true", "--verbose" });
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=2G,vcores=2", "--tensorboard", "true",
+ "--verbose" });
SubmarineStorage storage =
mockClientContext.getRuntimeFactory().getSubmarineStorage();
Map<String, String> jobInfo = storage.getJobInfoByName("my-job");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
index a195b59..b637036 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
@@ -53,7 +53,7 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager {
@Override
public Path getJobCheckpointDir(String jobName, boolean create)
throws IOException {
- return null;
+ return new Path("s3://generated_checkpoint_dir");
}
@Override
@@ -80,4 +80,9 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager {
public FileSystem getFileSystem() throws IOException {
return FileSystem.getLocal(new Configuration());
}
+
+ @Override
+ public Path getUserRootFolder() throws IOException {
+ return new Path("s3://generated_root_dir");
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDDS-458. numberofKeys is 0 for
all containers even when keys are present. Contributed by LiXin Ge.
Posted by xk...@apache.org.
HDDS-458. numberofKeys is 0 for all containers even when keys are present. Contributed by LiXin Ge.
(cherry picked from commit 8600b049af184af04dfb93b6bc353740e3e821d3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3c5221f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3c5221f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3c5221f
Branch: refs/heads/HDFS-12943
Commit: b3c5221f3091d394ef8c227f6776af07f7116658
Parents: 15ed74f
Author: Márton Elek <el...@apache.org>
Authored: Wed Sep 19 14:51:03 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Wed Sep 19 17:11:05 2018 +0200
----------------------------------------------------------------------
.../container/keyvalue/KeyValueContainer.java | 1 +
.../scm/container/TestContainerMapping.java | 46 ++++++++++++++++++++
2 files changed, 47 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3c5221f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index b893a38..0870c76 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -521,6 +521,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
.setWriteCount(containerData.getWriteCount())
.setReadBytes(containerData.getReadBytes())
.setWriteBytes(containerData.getWriteBytes())
+ .setKeyCount(containerData.getKeyCount())
.setUsed(containerData.getBytesUsed())
.setState(getHddsState())
.setDeleteTransactionId(containerData.getDeleteTransactionId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3c5221f/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
index 224f6ddd..f9a881e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
@@ -281,6 +281,52 @@ public class TestContainerMapping {
}
@Test
+ public void testListContainerAfterReport() throws Exception {
+ ContainerInfo info1 = createContainer();
+ ContainerInfo info2 = createContainer();
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
+ List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
+ new ArrayList<>();
+ StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
+ StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
+ long cID1 = info1.getContainerID();
+ long cID2 = info2.getContainerID();
+ ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
+ .setSize(1000000000L)
+ .setUsed(987654321L)
+ .setKeyCount(100000000L)
+ .setReadBytes(1000000000L)
+ .setWriteBytes(1000000000L)
+ .setContainerID(cID1);
+ reports.add(ciBuilder.build());
+
+ ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9")
+ .setSize(1000000000L)
+ .setUsed(123456789L)
+ .setKeyCount(200000000L)
+ .setReadBytes(3000000000L)
+ .setWriteBytes(4000000000L)
+ .setContainerID(cID2);
+ reports.add(ciBuilder.build());
+
+ ContainerReportsProto.Builder crBuilder = ContainerReportsProto
+ .newBuilder();
+ crBuilder.addAllReports(reports);
+
+ mapping.processContainerReports(datanodeDetails, crBuilder.build(), false);
+
+ List<ContainerInfo> list = mapping.listContainer(0, 50);
+ Assert.assertEquals(2, list.stream().filter(
+ x -> x.getContainerID() == cID1 || x.getContainerID() == cID2).count());
+ Assert.assertEquals(300000000L, list.stream().filter(
+ x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
+ .mapToLong(x -> x.getNumberOfKeys()).sum());
+ Assert.assertEquals(1111111110L, list.stream().filter(
+ x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
+ .mapToLong(x -> x.getUsedBytes()).sum());
+ }
+
+ @Test
public void testCloseContainer() throws IOException {
ContainerInfo info = createContainer();
mapping.updateContainerState(info.getContainerID(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-497. Suppress license
warnings for error log files. Contributed by Arpit Agarwal.
Posted by xk...@apache.org.
HDDS-497. Suppress license warnings for error log files. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb85351d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb85351d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb85351d
Branch: refs/heads/HDFS-12943
Commit: fb85351dc6506e92a7c8c3878d1897291b7850d0
Parents: 27978bc
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 21:18:15 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 21:18:15 2018 -0700
----------------------------------------------------------------------
hadoop-hdds/pom.xml | 7 ++++---
hadoop-ozone/pom.xml | 30 +++++++++++++++---------------
2 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb85351d/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 563ce9e..ce4f234 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -93,6 +93,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
+ <exclude>**/hs_err*.log</exclude>
<exclude>**/target/**</exclude>
<exclude>.gitattributes</exclude>
<exclude>.idea/**</exclude>
@@ -100,13 +101,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<exclude>src/main/resources/webapps/static/angular-nvd3-1.0.9.min.js</exclude>
<exclude>src/main/resources/webapps/static/angular-route-1.6.4.min.js</exclude>
<exclude>src/main/resources/webapps/static/d3-3.5.17.min.js</exclude>
- <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.css</exclude>
<exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.css.map</exclude>
- <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.js</exclude>
+ <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.css</exclude>
<exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.js.map</exclude>
- <exclude>src/test/resources/incorrect.container</exclude>
+ <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.js</exclude>
<exclude>src/test/resources/additionalfields.container</exclude>
<exclude>src/test/resources/incorrect.checksum.container</exclude>
+ <exclude>src/test/resources/incorrect.container</exclude>
<exclude>src/test/resources/test.db.ini</exclude>
</excludes>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb85351d/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 3c07b64..c73be60 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -120,37 +120,37 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/hs_err*.log</exclude>
<exclude>**/target/**</exclude>
<exclude>.gitattributes</exclude>
<exclude>.idea/**</exclude>
+ <exclude>dev-support/*tests</exclude>
<exclude>dev-support/checkstyle*</exclude>
<exclude>dev-support/jdiff/**</exclude>
- <exclude>dev-support/*tests</exclude>
- <exclude>src/test/empty-file</exclude>
- <exclude>src/test/all-tests</exclude>
- <exclude>src/test/resources/*.tgz</exclude>
- <exclude>src/test/resources/data*</exclude>
- <exclude>**/*.json</exclude>
- <exclude>src/test/resources/empty-file</exclude>
+ <exclude>src/contrib/**</exclude>
<exclude>src/main/webapps/datanode/robots.txt</exclude>
<exclude>src/main/webapps/hdfs/robots.txt</exclude>
<exclude>src/main/webapps/journal/robots.txt</exclude>
- <exclude>src/main/webapps/secondary/robots.txt</exclude>
<exclude>src/main/webapps/router/robots.txt</exclude>
- <exclude>src/contrib/**</exclude>
+ <exclude>src/main/webapps/secondary/robots.txt</exclude>
<exclude>src/site/resources/images/*</exclude>
+ <exclude>src/test/all-tests</exclude>
+ <exclude>src/test/empty-file</exclude>
+ <exclude>src/test/resources/*.tgz</exclude>
+ <exclude>src/test/resources/data*</exclude>
+ <exclude>src/test/resources/empty-file</exclude>
+ <exclude>webapps/static/angular-1.6.4.min.js</exclude>
+ <exclude>webapps/static/angular-nvd3-1.0.9.min.js</exclude>
+ <exclude>webapps/static/angular-route-1.6.4.min.js</exclude>
<exclude>webapps/static/bootstrap-3.3.7/**</exclude>
+ <exclude>webapps/static/d3-3.5.17.min.js</exclude>
<exclude>webapps/static/jquery-3.3.1.min.js</exclude>
<exclude>webapps/static/jquery.dataTables.min.js</exclude>
<exclude>webapps/static/nvd3-1.8.5.min.css.map</exclude>
- <exclude>webapps/static/nvd3-1.8.5.min.js</exclude>
- <exclude>webapps/static/angular-route-1.6.4.min.js
- </exclude>
<exclude>webapps/static/nvd3-1.8.5.min.css</exclude>
- <exclude>webapps/static/angular-nvd3-1.0.9.min.js</exclude>
<exclude>webapps/static/nvd3-1.8.5.min.js.map</exclude>
- <exclude>webapps/static/angular-1.6.4.min.js</exclude>
- <exclude>webapps/static/d3-3.5.17.min.js</exclude>
+ <exclude>webapps/static/nvd3-1.8.5.min.js</exclude>
</excludes>
</configuration>
</plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDDS-496. Ozone tools module is
incorrectly classified as 'hdds' component. Contributed by Dinesh Chitlangia.
Posted by xk...@apache.org.
HDDS-496. Ozone tools module is incorrectly classified as 'hdds' component. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c2ae7e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c2ae7e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c2ae7e4
Branch: refs/heads/HDFS-12943
Commit: 5c2ae7e493892b6157f73e82ca89c39926623bb1
Parents: a968ea4
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Tue Sep 18 14:50:52 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Tue Sep 18 14:51:03 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/tools/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c2ae7e4/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index 5296502..eeec595 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -29,7 +29,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>
<properties>
- <hadoop.component>hdds</hadoop.component>
+ <hadoop.component>ozone</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
</properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-13868. WebHDFS:
GETSNAPSHOTDIFF API NPE when param "snapshotname" is given but
"oldsnapshotname" is not. Contributed by Pranay Singh.
Posted by xk...@apache.org.
HDFS-13868. WebHDFS: GETSNAPSHOTDIFF API NPE when param "snapshotname" is given but "oldsnapshotname" is not. Contributed by Pranay Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28ceb34a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28ceb34a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28ceb34a
Branch: refs/heads/HDFS-12943
Commit: 28ceb34a725cd06d28fb51361c49bb45464f5368
Parents: e435e12
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Wed Sep 19 03:11:29 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Sep 19 03:12:20 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++++
.../namenode/snapshot/DirectorySnapshottableFeature.java | 2 +-
.../hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java | 6 ++++++
.../src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java | 6 ++++++
4 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index f4d11b9..38072b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2158,6 +2158,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
String fromSnapshot, String toSnapshot) throws IOException {
checkOpen();
try (TraceScope ignored = tracer.newScope("getSnapshotDiffReport")) {
+ Preconditions.checkArgument(fromSnapshot != null,
+ "null fromSnapshot");
+ Preconditions.checkArgument(toSnapshot != null,
+ "null toSnapshot");
return namenode
.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot);
} catch (RemoteException re) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index d3083cf..15aa22a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -277,7 +277,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
Snapshot fromSnapshot = getSnapshotByName(snapshotRootDir, from);
Snapshot toSnapshot = getSnapshotByName(snapshotRootDir, to);
// if the start point is equal to the end point, return null
- if (from.equals(to)) {
+ if (from != null && from.equals(to)) {
return null;
}
SnapshotDiffInfo diffs = new SnapshotDiffInfo(snapshotRootDir,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
index 4625988..18ec3c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
@@ -230,6 +230,12 @@ public class TestSnapshotDiffReport {
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
+ try {
+ report = hdfs.getSnapshotDiffReport(subsubsub1, null, "s2");
+ fail("Expect exception when providing null fromSnapshot ");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains("null fromSnapshot", e);
+ }
report = hdfs.getSnapshotDiffReport(subsubsub1, "s0", "s2");
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 5d33220..a766de6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -743,6 +743,12 @@ public class TestWebHDFS {
Assert.assertTrue(diffReport.getDiffList().contains(entry3));
Assert.assertTrue(diffReport.getDiffList().contains(entry4));
Assert.assertEquals(diffReport.getDiffList().size(), 5);
+
+ // Test with fromSnapshot and toSnapshot as null.
+ diffReport = webHdfs.getSnapshotDiffReport(foo, null, "s2");
+ Assert.assertEquals(diffReport.getDiffList().size(), 0);
+ diffReport = webHdfs.getSnapshotDiffReport(foo, "s1", null);
+ Assert.assertEquals(diffReport.getDiffList().size(), 5);
} finally {
if (cluster != null) {
cluster.shutdown();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HDFS-12943
Posted by xk...@apache.org.
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3810f7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3810f7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3810f7b
Branch: refs/heads/HDFS-12943
Commit: a3810f7b6bd822b4243a15c0ac31ab0f99b10b29
Parents: c377e3c 6b5838e
Author: Erik Krogen <xk...@apache.org>
Authored: Wed Sep 19 14:53:51 2018 -0700
Committer: Erik Krogen <xk...@apache.org>
Committed: Wed Sep 19 14:53:51 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 7 +-
.../hadoop-annotations/pom.xml | 24 ++
.../org/apache/hadoop/ipc/CallQueueManager.java | 5 +-
.../apache/hadoop/log/LogThrottlingHelper.java | 358 +++++++++++++++++++
.../org/apache/hadoop/util/StringUtils.java | 2 +-
.../hadoop/log/TestLogThrottlingHelper.java | 172 +++++++++
.../org/apache/hadoop/util/TestStringUtils.java | 9 +
.../main/compose/ozone-hdfs/docker-compose.yaml | 2 +-
.../src/main/compose/ozone-hdfs/docker-config | 1 +
.../src/main/compose/ozone/docker-config | 1 +
.../main/compose/ozonefs/docker-compose.yaml | 59 +++
.../src/main/compose/ozonefs/docker-config | 35 ++
.../src/main/compose/ozoneperf/docker-config | 1 +
.../src/main/compose/ozonescripts/docker-config | 1 +
hadoop-dist/src/main/smoketest/README.md | 30 ++
.../src/main/smoketest/basic/basic.robot | 47 +++
.../src/main/smoketest/basic/ozone-shell.robot | 82 +++++
hadoop-dist/src/main/smoketest/commonlib.robot | 24 ++
.../src/main/smoketest/ozonefs/ozonefs.robot | 35 ++
hadoop-dist/src/main/smoketest/test.sh | 101 ++++++
.../hadoop/hdds/scm/XceiverClientRatis.java | 6 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 +
.../scm/container/common/helpers/Pipeline.java | 22 +-
.../container/common/helpers/PipelineID.java | 13 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 14 +
.../main/java/org/apache/ratis/RatisHelper.java | 40 ++-
.../main/proto/ScmBlockLocationProtocol.proto | 6 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 7 +
.../common/src/main/resources/ozone-default.xml | 27 ++
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 21 ++
.../common/report/PipelineReportPublisher.java | 73 ++++
.../common/report/ReportPublisherFactory.java | 4 +
.../statemachine/DatanodeStateMachine.java | 19 +-
.../common/statemachine/StateContext.java | 42 ++-
.../CloseContainerCommandHandler.java | 17 +-
.../ReplicateContainerCommandHandler.java | 120 +------
.../states/datanode/InitDatanodeState.java | 12 +-
.../states/endpoint/RegisterEndpointTask.java | 8 +-
.../transport/server/XceiverServerGrpc.java | 28 +-
.../transport/server/XceiverServerSpi.java | 9 +
.../server/ratis/XceiverServerRatis.java | 105 +++---
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/ozoneimpl/OzoneContainer.java | 126 +++----
.../replication/ContainerReplicator.java | 27 ++
.../DownloadAndImportReplicator.java | 136 +++++++
.../replication/GrpcReplicationClient.java | 2 +-
.../replication/ReplicationSupervisor.java | 142 ++++++++
.../container/replication/ReplicationTask.java | 102 ++++++
.../StorageContainerDatanodeProtocol.java | 10 +-
.../protocol/StorageContainerNodeProtocol.java | 6 +-
.../commands/CloseContainerCommand.java | 23 +-
...rDatanodeProtocolClientSideTranslatorPB.java | 6 +-
...rDatanodeProtocolServerSideTranslatorPB.java | 5 +-
.../StorageContainerDatanodeProtocol.proto | 10 +
.../ozone/container/common/ScmTestMock.java | 8 +-
.../common/TestDatanodeStateMachine.java | 42 +++
.../TestReplicateContainerCommandHandler.java | 163 ---------
.../replication/TestReplicationSupervisor.java | 143 ++++++++
.../container/replication/package-info.java | 22 ++
.../hadoop/hdds/server/events/EventQueue.java | 18 +
hadoop-hdds/pom.xml | 7 +-
.../org/apache/hadoop/hdds/scm/ScmUtils.java | 45 +++
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 17 +-
.../container/CloseContainerEventHandler.java | 90 +++--
.../hdds/scm/container/ContainerMapping.java | 25 --
.../scm/container/ContainerReportHandler.java | 6 +-
.../hadoop/hdds/scm/container/Mapping.java | 15 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 15 +-
.../hdds/scm/exceptions/SCMException.java | 3 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 19 +-
.../hdds/scm/node/states/Node2ContainerMap.java | 123 +------
.../hdds/scm/node/states/Node2ObjectsMap.java | 162 +++++++++
.../hdds/scm/node/states/ReportResult.java | 105 +++---
.../hdds/scm/pipelines/Node2PipelineMap.java | 45 +--
.../scm/pipelines/PipelineCloseHandler.java | 24 +-
.../hdds/scm/pipelines/PipelineManager.java | 52 ++-
.../scm/pipelines/PipelineReportHandler.java | 59 +++
.../hdds/scm/pipelines/PipelineSelector.java | 107 +++---
.../scm/pipelines/ratis/RatisManagerImpl.java | 41 +--
.../standalone/StandaloneManagerImpl.java | 44 +--
.../hdds/scm/server/ChillModePrecheck.java | 12 +-
.../hdds/scm/server/SCMChillModeManager.java | 1 +
.../scm/server/SCMClientProtocolServer.java | 53 ++-
.../server/SCMDatanodeHeartbeatDispatcher.java | 23 ++
.../scm/server/SCMDatanodeProtocolServer.java | 16 +-
.../scm/server/StorageContainerManager.java | 129 +++++--
.../org/apache/hadoop/hdds/scm/TestUtils.java | 10 +-
.../hdds/scm/container/MockNodeManager.java | 4 +-
.../scm/container/TestContainerMapping.java | 54 ++-
.../hadoop/hdds/scm/node/TestNodeManager.java | 6 +-
.../scm/node/states/TestNode2ContainerMap.java | 28 +-
.../scm/server/TestSCMClientProtocolServer.java | 60 ++++
.../ozone/container/common/TestEndPoint.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 5 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 2 +-
.../hadoop/fs/http/server/FSOperations.java | 3 +
.../fs/http/client/BaseTestHttpFSWith.java | 35 +-
.../BlockPlacementPolicyDefault.java | 29 +-
.../apache/hadoop/hdfs/server/common/Util.java | 3 +-
.../hdfs/server/datanode/BlockSender.java | 7 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 46 +--
.../hdfs/server/namenode/ha/EditLogTailer.java | 34 +-
.../snapshot/DirectorySnapshottableFeature.java | 2 +-
.../apache/hadoop/hdfs/MiniDFSNNTopology.java | 17 +
.../blockmanagement/TestReplicationPolicy.java | 28 ++
.../TestDataNodeMultipleRegistrations.java | 4 +-
.../server/namenode/ha/TestEditLogTailer.java | 98 +++--
.../snapshot/TestSnapshotDiffReport.java | 6 +
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 6 +
.../v2/app/speculate/DefaultSpeculator.java | 4 +-
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 20 +-
hadoop-ozone/acceptance-test/README.md | 48 ---
.../dev-support/bin/robot-all.sh | 18 -
.../dev-support/bin/robot-dnd-all.sh | 57 ---
.../acceptance-test/dev-support/bin/robot.sh | 38 --
.../dev-support/docker/Dockerfile | 21 --
.../dev-support/docker/docker-compose.yaml | 23 --
hadoop-ozone/acceptance-test/pom.xml | 65 ----
.../src/test/acceptance/basic/.env | 17 -
.../src/test/acceptance/basic/basic.robot | 50 ---
.../test/acceptance/basic/docker-compose.yaml | 50 ---
.../src/test/acceptance/basic/docker-config | 34 --
.../src/test/acceptance/basic/ozone-shell.robot | 86 -----
.../src/test/acceptance/commonlib.robot | 78 ----
.../src/test/acceptance/ozonefs/.env | 17 -
.../test/acceptance/ozonefs/docker-compose.yaml | 59 ---
.../src/test/acceptance/ozonefs/docker-config | 35 --
.../src/test/acceptance/ozonefs/ozonefs.robot | 39 --
.../acceptance/ozonefs/ozonesinglenode.robot | 49 ---
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +-
.../src/main/proto/OzoneManagerProtocol.proto | 2 +-
hadoop-ozone/docs/content/JavaApi.md | 4 +
hadoop-ozone/docs/content/OzoneFS.md | 6 +-
hadoop-ozone/docs/pom.xml | 5 +
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +-
.../hdds/scm/pipeline/TestSCMRestart.java | 20 +-
.../apache/hadoop/ozone/RatisTestHelper.java | 9 +
.../ozone/TestStorageContainerManager.java | 54 +++
.../rpc/TestCloseContainerHandlingByClient.java | 5 +-
.../TestCloseContainerByPipeline.java | 5 +-
.../hadoop/ozone/om/TestScmChillMode.java | 171 +++++++++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 15 +-
.../hadoop/ozone/web/client/TestKeys.java | 1 +
.../hadoop/ozone/web/client/TestKeysRatis.java | 2 -
.../src/test/resources/log4j.properties | 5 +-
.../org/apache/hadoop/ozone/TestErrorCode.java | 53 ---
.../apache/hadoop/ozone/web/TestErrorCode.java | 53 +++
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 32 +-
.../apache/hadoop/ozone/om/OzoneManager.java | 33 +-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../hadoop/ozone/om/TestKeyManagerImpl.java | 165 +++++++++
hadoop-ozone/ozonefs/pom.xml | 14 +-
.../ozonefs/src/test/resources/log4j.properties | 1 +
hadoop-ozone/pom.xml | 31 +-
hadoop-ozone/tools/pom.xml | 2 +-
hadoop-project-dist/pom.xml | 34 +-
hadoop-project/pom.xml | 6 +-
.../yarn/submarine/client/cli/CliConstants.java | 6 +
.../yarn/submarine/client/cli/CliUtils.java | 10 +-
.../yarn/submarine/client/cli/RunJobCli.java | 43 ++-
.../client/cli/param/RunJobParameters.java | 52 ++-
.../fs/DefaultRemoteDirectoryManager.java | 21 +-
.../common/fs/RemoteDirectoryManager.java | 4 +-
.../common/FSBasedSubmarineStorageImpl.java | 4 +-
.../yarnservice/YarnServiceJobSubmitter.java | 126 ++++---
.../runtimes/yarnservice/YarnServiceUtils.java | 26 +-
.../yarnservice/TestYarnServiceRunJobCli.java | 268 ++++++++++++--
.../common/fs/MockRemoteDirectoryManager.java | 7 +-
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../launcher/ContainerLaunch.java | 5 +-
.../linux/resources/ResourceHandlerModule.java | 15 +
.../runtime/DockerLinuxContainerRuntime.java | 5 +-
.../linux/runtime/docker/DockerRmCommand.java | 11 +-
.../impl/container-executor.c | 153 +++++++-
.../impl/container-executor.h | 8 +-
.../main/native/container-executor/impl/main.c | 12 +-
.../test/test-container-executor.c | 147 ++++++++
.../launcher/TestContainerLaunch.java | 81 +++++
.../docker/TestDockerCommandExecutor.java | 23 +-
.../runtime/docker/TestDockerRmCommand.java | 35 +-
.../allocator/RegularContainerAllocator.java | 3 +-
.../capacity/TestContainerAllocation.java | 68 ++++
.../site/markdown/PlacementConstraints.md.vm | 2 +
.../main/webapp/app/controllers/application.js | 5 +-
.../src/main/webapp/app/initializers/loader.js | 3 +
.../src/main/webapp/app/routes/application.js | 4 +-
pom.xml | 3 +-
192 files changed, 4906 insertions(+), 2132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3810f7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3810f7b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-15742. Log if ipc backoff
is enabled in CallQueueManager. Contributed by Ryan Wu.
Posted by xk...@apache.org.
HADOOP-15742. Log if ipc backoff is enabled in CallQueueManager. Contributed by Ryan Wu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee051ef9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee051ef9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee051ef9
Branch: refs/heads/HDFS-12943
Commit: ee051ef9fec1fddb612aa1feae9fd3df7091354f
Parents: 281c192
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Sep 18 11:10:33 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Sep 18 11:10:33 2018 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ipc/CallQueueManager.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee051ef9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
index d1bd180..29649a6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
@@ -81,8 +81,9 @@ public class CallQueueManager<E extends Schedulable>
this.clientBackOffEnabled = clientBackOffEnabled;
this.putRef = new AtomicReference<BlockingQueue<E>>(bq);
this.takeRef = new AtomicReference<BlockingQueue<E>>(bq);
- LOG.info("Using callQueue: " + backingClass + " queueCapacity: " +
- maxQueueSize + " scheduler: " + schedulerClass);
+ LOG.info("Using callQueue: {}, queueCapacity: {}, " +
+ "scheduler: {}, ipcBackoff: {}.",
+ backingClass, maxQueueSize, schedulerClass, clientBackOffEnabled);
}
@VisibleForTesting // only!
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: MAPREDUCE-3801:
org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator
fails intermittently. Contributed by Jason Lowe
Posted by xk...@apache.org.
MAPREDUCE-3801: org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2cf89276
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2cf89276
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2cf89276
Branch: refs/heads/HDFS-12943
Commit: 2cf8927683d49f4cdc3cba038480128c473c5461
Parents: f938925
Author: Eric E Payne <er...@oath.com>
Authored: Tue Sep 18 20:26:28 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Tue Sep 18 20:26:28 2018 +0000
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cf89276/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
index fa65383..6573687 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
@@ -100,8 +100,6 @@ public class DefaultSpeculator extends AbstractService implements
private AppContext context;
private Thread speculationBackgroundThread = null;
private volatile boolean stopped = false;
- private BlockingQueue<SpeculatorEvent> eventQueue
- = new LinkedBlockingQueue<SpeculatorEvent>();
private TaskRuntimeEstimator estimator;
private BlockingQueue<Object> scanControl = new LinkedBlockingQueue<Object>();
@@ -247,7 +245,7 @@ public class DefaultSpeculator extends AbstractService implements
// This section is not part of the Speculator interface; it's used only for
// testing
public boolean eventQueueEmpty() {
- return eventQueue.isEmpty();
+ return scanControl.isEmpty();
}
// This interface is intended to be used only for test cases.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: MAPREDUCE-7138.
ThrottledContainerAllocator in MRAppBenchmark should implement
RMHeartbeatHandler. Contributed by Oleksandr Shevchenko
Posted by xk...@apache.org.
MAPREDUCE-7138. ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler. Contributed by Oleksandr Shevchenko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8382b860
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8382b860
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8382b860
Branch: refs/heads/HDFS-12943
Commit: 8382b860d4ef4f20d000537ded42a88e98bd2190
Parents: 34b2237
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 17:06:32 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 17:06:32 2018 -0500
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8382b860/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
index 5e6697b..efe150f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
@@ -33,10 +33,12 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssigned
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -116,7 +118,7 @@ public class MRAppBenchmark {
}
class ThrottledContainerAllocator extends AbstractService
- implements ContainerAllocator {
+ implements ContainerAllocator, RMHeartbeatHandler {
private int containerCount;
private Thread thread;
private BlockingQueue<ContainerAllocatorEvent> eventQueue =
@@ -182,6 +184,15 @@ public class MRAppBenchmark {
}
super.serviceStop();
}
+
+ @Override
+ public long getLastHeartbeatTime() {
+ return Time.now();
+ }
+
+ @Override
+ public void runOnNextHeartbeat(Runnable callback) {
+ }
}
}
@@ -264,7 +275,7 @@ public class MRAppBenchmark {
});
}
- @Test
+ @Test(timeout = 60000)
public void benchmark2() throws Exception {
int maps = 100; // Adjust for benchmarking, start with a couple of thousands
int reduces = 50;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-8652. [UI2] YARN UI2 breaks
if getUserInfo REST API is not available in older versions. Contributed by
Akhil PB.
Posted by xk...@apache.org.
YARN-8652. [UI2] YARN UI2 breaks if getUserInfo REST API is not available in older versions. Contributed by Akhil PB.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbeca010
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbeca010
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbeca010
Branch: refs/heads/HDFS-12943
Commit: bbeca0107e247ae14cfe96761f9e5fbb1f02e53d
Parents: 0cc6e03
Author: Sunil G <su...@apache.org>
Authored: Tue Sep 18 12:44:21 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Sep 18 12:44:21 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/controllers/application.js | 5 ++++-
.../hadoop-yarn-ui/src/main/webapp/app/routes/application.js | 4 +++-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbeca010/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
index 75b072a..3e8fcce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
@@ -65,6 +65,9 @@ export default Ember.Controller.extend({
}.property('model.clusterInfo'),
userInfo: function() {
- return this.model.userInfo.get('firstObject');
+ if (this.model.userInfo) {
+ return this.model.userInfo.get('firstObject');
+ }
+ return null;
}.property('model.userInfo'),
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbeca010/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
index e30baaa..60d1efd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
@@ -23,7 +23,9 @@ export default AbstractRoute.extend({
model() {
return Ember.RSVP.hash({
clusterInfo: this.store.findAll('ClusterInfo', {reload: true}),
- userInfo: this.store.findAll('cluster-user-info', {reload: true})
+ userInfo: this.store.findAll('cluster-user-info', {reload: true}).catch(function() {
+ return null;
+ })
});
},
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-461. Container remains in
CLOSING state in SCM forever. Contributed by Shashikant Banerjee.
Posted by xk...@apache.org.
HDDS-461. Container remains in CLOSING state in SCM forever. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61a4b07b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61a4b07b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61a4b07b
Branch: refs/heads/HDFS-12943
Commit: 61a4b07bdaf68d45c7c0bba0485d95a1d908a73d
Parents: b3c5221
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 21:43:44 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 21:43:44 2018 +0530
----------------------------------------------------------------------
.../common/statemachine/StateContext.java | 42 +++++--
.../CloseContainerCommandHandler.java | 17 ++-
.../transport/server/XceiverServerGrpc.java | 12 +-
.../server/ratis/XceiverServerRatis.java | 31 +++--
.../container/ozoneimpl/OzoneContainer.java | 116 ++++++-------------
.../commands/CloseContainerCommand.java | 23 +---
.../container/CloseContainerEventHandler.java | 90 ++++++++------
.../TestCloseContainerByPipeline.java | 5 +-
8 files changed, 175 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 9d5a778..47c2492 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine;
+import com.google.common.base.Preconditions;
import com.google.protobuf.GeneratedMessage;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -43,6 +44,7 @@ import org.slf4j.LoggerFactory;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
+import java.util.ArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -50,7 +52,6 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
@@ -206,9 +207,18 @@ public class StateContext {
* @return List<reports>
*/
public List<GeneratedMessage> getReports(int maxLimit) {
+ List<GeneratedMessage> reportList = new ArrayList<>();
synchronized (reports) {
- return reports.parallelStream().limit(maxLimit)
- .collect(Collectors.toList());
+ if (!reports.isEmpty()) {
+ int size = reports.size();
+ int limit = size > maxLimit ? maxLimit : size;
+ for (int count = 0; count < limit; count++) {
+ GeneratedMessage report = reports.poll();
+ Preconditions.checkNotNull(report);
+ reportList.add(report);
+ }
+ }
+ return reportList;
}
}
@@ -254,9 +264,20 @@ public class StateContext {
* @return List<ContainerAction>
*/
public List<ContainerAction> getPendingContainerAction(int maxLimit) {
+ List<ContainerAction> containerActionList = new ArrayList<>();
synchronized (containerActions) {
- return containerActions.parallelStream().limit(maxLimit)
- .collect(Collectors.toList());
+ if (!containerActions.isEmpty()) {
+ int size = containerActions.size();
+ int limit = size > maxLimit ? maxLimit : size;
+ for (int count = 0; count < limit; count++) {
+ // we need to remove the action from the containerAction queue
+ // as well
+ ContainerAction action = containerActions.poll();
+ Preconditions.checkNotNull(action);
+ containerActionList.add(action);
+ }
+ }
+ return containerActionList;
}
}
@@ -295,9 +316,16 @@ public class StateContext {
* @return List<ContainerAction>
*/
public List<PipelineAction> getPendingPipelineAction(int maxLimit) {
+ List<PipelineAction> pipelineActionList = new ArrayList<>();
synchronized (pipelineActions) {
- return pipelineActions.parallelStream().limit(maxLimit)
- .collect(Collectors.toList());
+ if (!pipelineActions.isEmpty()) {
+ int size = pipelineActions.size();
+ int limit = size > maxLimit ? maxLimit : size;
+ for (int count = 0; count < limit; count++) {
+ pipelineActionList.add(pipelineActions.poll());
+ }
+ }
+ return pipelineActionList;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index 030a357..d4e13ee 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
+import org.apache.ratis.protocol.NotLeaderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,9 +90,21 @@ public class CloseContainerCommandHandler implements CommandHandler {
// submit the close container request for the XceiverServer to handle
container.submitContainerRequest(
request.build(), replicationType, pipelineID);
- cmdExecuted = true;
} catch (Exception e) {
- LOG.error("Can't close container " + containerID, e);
+ if (e instanceof NotLeaderException) {
+ // If the particular datanode is not the Ratis leader, the close
+ // container command will not be executed by the follower but will be
+ // executed by Ratis stateMachine transactions via leader to follower.
+ // There can also be case where the datanode is in candidate state.
+ // In these situations, NotLeaderException is thrown. Remove the status
+ // from cmdStatus Map here so that it will be retried only by SCM if the
+ // leader could not not close the container after a certain time.
+ context.removeCommandStatus(containerID);
+ LOG.info(e.getLocalizedMessage());
+ } else {
+ LOG.error("Can't close container " + containerID, e);
+ cmdExecuted = false;
+ }
} finally {
updateCommandStatus(context, command, cmdExecuted, LOG);
long endTime = Time.monotonicNow();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 83e742c..c51da98 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -21,12 +21,15 @@ package org.apache.hadoop.ozone.container.common.transport.server;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.
+ StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -128,8 +131,13 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
@Override
public void submitRequest(ContainerCommandRequestProto request,
- HddsProtos.PipelineID pipelineID) {
- storageContainer.dispatch(request);
+ HddsProtos.PipelineID pipelineID) throws IOException {
+ ContainerProtos.ContainerCommandResponseProto response =
+ storageContainer.dispatch(request);
+ if (response.getResult() != ContainerProtos.Result.SUCCESS) {
+ throw new StorageContainerException(response.getMessage(),
+ response.getResult());
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index d88995b..c2ef504 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -74,7 +74,6 @@ import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -319,39 +318,35 @@ public final class XceiverServerRatis implements XceiverServerSpi {
return server;
}
- private void processReply(RaftClientReply reply) {
-
+ private void processReply(RaftClientReply reply) throws IOException {
// NotLeader exception is thrown only when the raft server to which the
// request is submitted is not the leader. The request will be rejected
- // and will eventually be executed once the request comnes via the leader
+ // and will eventually be executed once the request comes via the leader
// node.
NotLeaderException notLeaderException = reply.getNotLeaderException();
if (notLeaderException != null) {
- LOG.info(reply.getNotLeaderException().getLocalizedMessage());
+ throw notLeaderException;
}
StateMachineException stateMachineException =
reply.getStateMachineException();
if (stateMachineException != null) {
- // In case the request could not be completed, StateMachine Exception
- // will be thrown. For now, Just log the message.
- // If the container could not be closed, SCM will come to know
- // via containerReports. CloseContainer should be re tried via SCM.
- LOG.error(stateMachineException.getLocalizedMessage());
+ throw stateMachineException;
}
}
@Override
- public void submitRequest(
- ContainerCommandRequestProto request, HddsProtos.PipelineID pipelineID)
- throws IOException {
- // ReplicationLevel.MAJORITY ensures the transactions corresponding to
- // the request here are applied on all the raft servers.
+ public void submitRequest(ContainerCommandRequestProto request,
+ HddsProtos.PipelineID pipelineID) throws IOException {
+ RaftClientReply reply;
RaftClientRequest raftClientRequest =
createRaftClientRequest(request, pipelineID,
RaftClientRequest.writeRequestType(replicationLevel));
- CompletableFuture<RaftClientReply> reply =
- server.submitClientRequestAsync(raftClientRequest);
- reply.thenAccept(this::processReply);
+ try {
+ reply = server.submitClientRequestAsync(raftClientRequest).get();
+ } catch (Exception e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ processReply(reply);
}
private RaftClientRequest createRaftClientRequest(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index ebacf75..da58772 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -23,7 +23,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -46,12 +47,14 @@ import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
/**
- * Ozone main class sets up the network server and initializes the container
+ * Ozone main class sets up the network servers and initializes the container
* layer.
*/
public class OzoneContainer {
@@ -64,7 +67,7 @@ public class OzoneContainer {
private final OzoneConfiguration config;
private final VolumeSet volumeSet;
private final ContainerSet containerSet;
- private final XceiverServerSpi[] server;
+ private final Map<ReplicationType, XceiverServerSpi> servers;
/**
* Construct OzoneContainer object.
@@ -82,14 +85,13 @@ public class OzoneContainer {
buildContainerSet();
hddsDispatcher = new HddsDispatcher(config, containerSet, volumeSet,
context);
- server = new XceiverServerSpi[]{
- new XceiverServerGrpc(datanodeDetails, this.config, this
- .hddsDispatcher, createReplicationService()),
- XceiverServerRatis.newXceiverServerRatis(datanodeDetails, this
- .config, hddsDispatcher, context)
- };
-
-
+ servers = new HashMap<>();
+ servers.put(ReplicationType.STAND_ALONE,
+ new XceiverServerGrpc(datanodeDetails, config, hddsDispatcher,
+ createReplicationService()));
+ servers.put(ReplicationType.RATIS, XceiverServerRatis
+ .newXceiverServerRatis(datanodeDetails, config, hddsDispatcher,
+ context));
}
private GrpcReplicationService createReplicationService() {
@@ -133,7 +135,7 @@ public class OzoneContainer {
*/
public void start() throws IOException {
LOG.info("Attempting to start container services.");
- for (XceiverServerSpi serverinstance : server) {
+ for (XceiverServerSpi serverinstance : servers.values()) {
serverinstance.start();
}
hddsDispatcher.init();
@@ -145,7 +147,7 @@ public class OzoneContainer {
public void stop() {
//TODO: at end of container IO integration work.
LOG.info("Attempting to stop container services.");
- for(XceiverServerSpi serverinstance: server) {
+ for(XceiverServerSpi serverinstance: servers.values()) {
serverinstance.stop();
}
hddsDispatcher.shutdown();
@@ -169,7 +171,7 @@ public class OzoneContainer {
public PipelineReportsProto getPipelineReport() {
PipelineReportsProto.Builder pipelineReportsProto =
PipelineReportsProto.newBuilder();
- for (XceiverServerSpi serverInstance : server) {
+ for (XceiverServerSpi serverInstance : servers.values()) {
pipelineReportsProto
.addAllPipelineReport(serverInstance.getPipelineReport());
}
@@ -181,82 +183,38 @@ public class OzoneContainer {
* @param request
* @param replicationType
* @param pipelineID
- * @throws IOException
*/
public void submitContainerRequest(
ContainerProtos.ContainerCommandRequestProto request,
- HddsProtos.ReplicationType replicationType,
- HddsProtos.PipelineID pipelineID) throws IOException {
- XceiverServerSpi serverInstance;
- long containerId = getContainerIdForCmd(request);
- if (replicationType == HddsProtos.ReplicationType.RATIS) {
- serverInstance = getRatisSerer();
- Preconditions.checkNotNull(serverInstance);
- serverInstance.submitRequest(request, pipelineID);
- LOG.info("submitting {} request over RATIS server for container {}",
- request.getCmdType(), containerId);
- } else {
- serverInstance = getStandaAloneSerer();
- Preconditions.checkNotNull(serverInstance);
- getStandaAloneSerer().submitRequest(request, pipelineID);
- LOG.info(
- "submitting {} request over STAND_ALONE server for container {}",
- request.getCmdType(), containerId);
+ ReplicationType replicationType,
+ PipelineID pipelineID) throws IOException {
+ if (containerSet.getContainer(request.getContainerID())
+ .getContainerData().isClosed()) {
+ LOG.debug("Container {} is already closed", request.getContainerID());
+ // It might happen that the where the first attempt of closing the
+ // container failed with NOT_LEADER_EXCEPTION. In such cases, SCM will
+ // retry to check the container got really closed via Ratis.
+ // In such cases of the retry attempt, if the container is already closed
+ // via Ratis, we should just return.
}
-
+ LOG.info("submitting {} request over {} server for container {}",
+ request.getCmdType(), replicationType, request.getContainerID());
+ Preconditions.checkState(servers.containsKey(replicationType));
+ servers.get(replicationType).submitRequest(request, pipelineID);
}
- private long getContainerIdForCmd(
- ContainerProtos.ContainerCommandRequestProto request)
- throws IllegalArgumentException {
- ContainerProtos.Type type = request.getCmdType();
- switch (type) {
- case CloseContainer:
- return request.getContainerID();
- // Right now, we handle only closeContainer via queuing it over the
- // over the XceiVerServer. For all other commands we throw Illegal
- // argument exception here. Will need to extend the switch cases
- // in case we want add another commands here.
- default:
- throw new IllegalArgumentException("Cmd " + request.getCmdType()
- + " not supported over HearBeat Response");
- }
- }
-
- private XceiverServerSpi getRatisSerer() {
- for (XceiverServerSpi serverInstance : server) {
- if (serverInstance instanceof XceiverServerRatis) {
- return serverInstance;
- }
- }
- return null;
- }
-
- private XceiverServerSpi getStandaAloneSerer() {
- for (XceiverServerSpi serverInstance : server) {
- if (!(serverInstance instanceof XceiverServerRatis)) {
- return serverInstance;
- }
- }
- return null;
- }
-
- private int getPortbyType(HddsProtos.ReplicationType replicationType) {
- for (XceiverServerSpi serverinstance : server) {
- if (serverinstance.getServerType() == replicationType) {
- return serverinstance.getIPCPort();
- }
- }
- return INVALID_PORT;
+ private int getPortByType(ReplicationType replicationType) {
+ return servers.containsKey(replicationType) ?
+ servers.get(replicationType).getIPCPort() : INVALID_PORT;
}
/**
- * Returns the container server IPC port.
+ * Returns the container servers IPC port.
*
- * @return Container server IPC port.
+ * @return Container servers IPC port.
*/
public int getContainerServerPort() {
- return getPortbyType(HddsProtos.ReplicationType.STAND_ALONE);
+ return getPortByType(ReplicationType.STAND_ALONE);
}
/**
@@ -265,7 +223,7 @@ public class OzoneContainer {
* @return Ratis port.
*/
public int getRatisContainerServerPort() {
- return getPortbyType(HddsProtos.ReplicationType.RATIS);
+ return getPortByType(ReplicationType.RATIS);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
index aaa5f11..c2c20a4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
@@ -30,25 +30,13 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
public class CloseContainerCommand
extends SCMCommand<CloseContainerCommandProto> {
- private long containerID;
private HddsProtos.ReplicationType replicationType;
private PipelineID pipelineID;
public CloseContainerCommand(long containerID,
HddsProtos.ReplicationType replicationType,
PipelineID pipelineID) {
- super();
- this.containerID = containerID;
- this.replicationType = replicationType;
- this.pipelineID = pipelineID;
- }
-
- // Should be called only for protobuf conversion
- private CloseContainerCommand(long containerID,
- HddsProtos.ReplicationType replicationType,
- PipelineID pipelineID, long id) {
- super(id);
- this.containerID = containerID;
+ super(containerID);
this.replicationType = replicationType;
this.pipelineID = pipelineID;
}
@@ -75,7 +63,7 @@ public class CloseContainerCommand
public CloseContainerCommandProto getProto() {
return CloseContainerCommandProto.newBuilder()
- .setContainerID(containerID)
+ .setContainerID(getId())
.setCmdId(getId())
.setReplicationType(replicationType)
.setPipelineID(pipelineID.getProtobuf())
@@ -85,13 +73,12 @@ public class CloseContainerCommand
public static CloseContainerCommand getFromProtobuf(
CloseContainerCommandProto closeContainerProto) {
Preconditions.checkNotNull(closeContainerProto);
- return new CloseContainerCommand(closeContainerProto.getContainerID(),
+ return new CloseContainerCommand(closeContainerProto.getCmdId(),
closeContainerProto.getReplicationType(),
- PipelineID.getFromProtobuf(closeContainerProto.getPipelineID()),
- closeContainerProto.getCmdId());
+ PipelineID.getFromProtobuf(closeContainerProto.getPipelineID()));
}
public long getContainerID() {
- return containerID;
+ return getId();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
index b94ce4f..7baecc4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
@@ -17,10 +17,10 @@
package org.apache.hadoop.hdds.scm.container;
import java.io.IOException;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.IdentifiableEventPayload;
@@ -57,7 +57,7 @@ public class CloseContainerEventHandler implements EventHandler<ContainerID> {
LOG.info("Close container Event triggered for container : {}",
containerID.getId());
- ContainerWithPipeline containerWithPipeline = null;
+ ContainerWithPipeline containerWithPipeline;
ContainerInfo info;
try {
containerWithPipeline =
@@ -74,42 +74,66 @@ public class CloseContainerEventHandler implements EventHandler<ContainerID> {
return;
}
- if (info.getState() == HddsProtos.LifeCycleState.OPEN) {
- for (DatanodeDetails datanode :
- containerWithPipeline.getPipeline().getMachines()) {
- CommandForDatanode closeContainerCommand = new CommandForDatanode<>(
- datanode.getUuid(),
- new CloseContainerCommand(containerID.getId(),
- info.getReplicationType(), info.getPipelineID()));
- publisher.fireEvent(DATANODE_COMMAND, closeContainerCommand);
- publisher.fireEvent(CLOSE_CONTAINER_RETRYABLE_REQ, new
- CloseContainerRetryableReq(containerID));
- }
- try {
- // Finalize event will make sure the state of the container transitions
- // from OPEN to CLOSING in containerStateManager.
- containerManager.updateContainerState(containerID.getId(),
- HddsProtos.LifeCycleEvent.FINALIZE);
- } catch (IOException ex) {
- LOG.error("Failed to update the container state to FINALIZE for"
- + "container : {}" + containerID, ex);
- }
- } else if (info.getState() == HddsProtos.LifeCycleState.ALLOCATED) {
- try {
- // Create event will make sure the state of the container transitions
- // from OPEN to CREATING in containerStateManager, this will move
- // the container out of active allocation path.
+ HddsProtos.LifeCycleState state = info.getState();
+ try {
+ switch (state) {
+ case ALLOCATED:
+ // We cannot close a container in ALLOCATED state, moving the
+ // container to CREATING state, this should eventually
+ // timeout and the container will be moved to DELETING state.
+ LOG.debug("Closing container {} in {} state", containerID, state);
containerManager.updateContainerState(containerID.getId(),
HddsProtos.LifeCycleEvent.CREATE);
- } catch (IOException ex) {
- LOG.error("Failed to update the container state to CREATE for"
- + "container:{}" + containerID, ex);
+ break;
+ case CREATING:
+ // We cannot close a container in CREATING state, it will eventually
+ // timeout and moved to DELETING state.
+ LOG.debug("Closing container {} in {} state", containerID, state);
+ break;
+ case OPEN:
+ containerManager.updateContainerState(containerID.getId(),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ fireCloseContainerEvents(containerWithPipeline, info, publisher);
+ break;
+ case CLOSING:
+ fireCloseContainerEvents(containerWithPipeline, info, publisher);
+ break;
+ case CLOSED:
+ case DELETING:
+ case DELETED:
+ LOG.info(
+ "container with id : {} is in {} state and need not be closed.",
+ containerID.getId(), info.getState());
+ break;
+ default:
+ throw new IOException(
+ "Invalid container state for container " + containerID);
}
- } else {
- LOG.info("container with id : {} is in {} state and need not be closed.",
- containerID.getId(), info.getState());
+ } catch (IOException ex) {
+ LOG.error("Failed to update the container state for" + "container : {}"
+ + containerID, ex);
}
+ }
+
+ private void fireCloseContainerEvents(
+ ContainerWithPipeline containerWithPipeline, ContainerInfo info,
+ EventPublisher publisher) {
+ ContainerID containerID = info.containerID();
+ // fire events.
+ CloseContainerCommand closeContainerCommand =
+ new CloseContainerCommand(containerID.getId(),
+ info.getReplicationType(), info.getPipelineID());
+ Pipeline pipeline = containerWithPipeline.getPipeline();
+ pipeline.getMachines().stream().map(
+ datanode -> new CommandForDatanode<>(datanode.getUuid(),
+ closeContainerCommand)).forEach((command) -> {
+ publisher.fireEvent(DATANODE_COMMAND, command);
+ });
+ publisher.fireEvent(CLOSE_CONTAINER_RETRYABLE_REQ,
+ new CloseContainerRetryableReq(containerID));
+ LOG.trace("Issuing {} on Pipeline {} for container", closeContainerCommand,
+ pipeline, containerID);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index ed9c54d..8c52847 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -132,7 +132,6 @@ public class TestCloseContainerByPipeline {
// Make sure the closeContainerCommandHandler is Invoked
Assert.assertTrue(
closeContainerHandler.getInvocationCount() > lastInvocationCount);
-
}
@Test
@@ -190,6 +189,7 @@ public class TestCloseContainerByPipeline {
Assert.assertFalse((logCapturer.getOutput().contains(
"submitting CloseContainer request over RATIS server for container "
+ containerID)));
+ logCapturer.stopCapturing();
}
@Test
@@ -239,13 +239,14 @@ public class TestCloseContainerByPipeline {
Assert.assertTrue(isContainerClosed(cluster,
containerID, datanodeDetails));
}
+ // Make sure it was really closed via Ratis not STAND_ALONE server
Assert.assertFalse(logCapturer.getOutput().contains(
"submitting CloseContainer request over STAND_ALONE "
+ "server for container " + containerID));
- // Make sure it was really closed via StandAlone not Ratis server
Assert.assertTrue((logCapturer.getOutput().contains(
"submitting CloseContainer request over RATIS server for container "
+ containerID)));
+ logCapturer.stopCapturing();
}
private Boolean isContainerClosed(MiniOzoneCluster cluster, long containerID,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDDS-507. EventQueue should be
shutdown on SCM shutdown. Contributed by Xiaoyu Yao.
Posted by xk...@apache.org.
HDDS-507. EventQueue should be shutdown on SCM shutdown. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/236d16e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/236d16e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/236d16e3
Branch: refs/heads/HDFS-12943
Commit: 236d16e3a5611b8ca4f80cb7e8ffd63f167db8e0
Parents: 042bf74
Author: Nanda kumar <na...@apache.org>
Authored: Thu Sep 20 01:22:18 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Thu Sep 20 01:22:18 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/server/StorageContainerManager.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/236d16e3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 86c061b..b2cbc93 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -759,6 +759,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
unregisterMXBean();
+ // Event queue must be stopped before the DB store is closed at the end.
+ try {
+ LOG.info("Stopping SCM Event Queue.");
+ eventQueue.close();
+ } catch (Exception ex) {
+ LOG.error("SCM Event Queue stop failed", ex);
+ }
IOUtils.cleanupWithLogger(LOG, scmContainerManager);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDDS-506. Fields in
AllocateScmBlockResponseProto should be optional. Contributed by Arpit
Agarwal.
Posted by xk...@apache.org.
HDDS-506. Fields in AllocateScmBlockResponseProto should be optional. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e435e12f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e435e12f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e435e12f
Branch: refs/heads/HDFS-12943
Commit: e435e12f1fd00a5f78621ec1933a60171fc224e6
Parents: fb85351
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Sep 18 23:21:29 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Sep 18 23:21:29 2018 -0700
----------------------------------------------------------------------
.../common/src/main/proto/ScmBlockLocationProtocol.proto | 6 +++---
.../org/apache/hadoop/hdds/scm/exceptions/SCMException.java | 3 ++-
.../org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java | 4 ++++
3 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e435e12f/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
index 53f408a..9b4e0ac 100644
--- a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
@@ -104,9 +104,9 @@ message AllocateScmBlockResponseProto {
unknownFailure = 4;
}
required Error errorCode = 1;
- required BlockID blockID = 2;
- required hadoop.hdds.Pipeline pipeline = 3;
- required bool createContainer = 4;
+ optional BlockID blockID = 2;
+ optional hadoop.hdds.Pipeline pipeline = 3;
+ optional bool createContainer = 4;
optional string errorMessage = 5;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e435e12f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
index 87a29e3..dae0b06 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
@@ -118,6 +118,7 @@ public class SCMException extends IOException {
SCM_NOT_INITIALIZED,
DUPLICATE_DATANODE,
NO_SUCH_DATANODE,
- NO_REPLICA_FOUND
+ NO_REPLICA_FOUND,
+ FAILED_TO_FIND_ACTIVE_PIPELINE
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e435e12f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
index c9f51f7..82946bd 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
@@ -62,6 +62,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
.FAILED_TO_CHANGE_PIPELINE_STATE;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.FAILED_TO_FIND_ACTIVE_PIPELINE;
import static org.apache.hadoop.hdds.server
.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone
@@ -285,6 +286,9 @@ public class PipelineSelector {
// try to return a pipeline from already allocated pipelines
PipelineID pipelineId =
manager.getPipeline(replicationFactor, replicationType);
+ if (pipelineId == null) {
+ throw new SCMException(FAILED_TO_FIND_ACTIVE_PIPELINE);
+ }
pipeline = pipelineMap.get(pipelineId);
Preconditions.checkArgument(pipeline.getLifeCycleState() ==
LifeCycleState.OPEN);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDDS-476. Add Pipeline reports to
make pipeline active on SCM restart. Contributed by Mukul Kumar Singh.
Posted by xk...@apache.org.
HDDS-476. Add Pipeline reports to make pipeline active on SCM restart.
Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0956ee2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0956ee2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0956ee2
Branch: refs/heads/HDFS-12943
Commit: c0956ee2a879d1f82938dd2b8bab79b09ae32eac
Parents: 0712537e
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 18:49:13 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 18:52:08 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientRatis.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +
.../scm/container/common/helpers/Pipeline.java | 22 ++-
.../container/common/helpers/PipelineID.java | 13 +-
.../common/src/main/resources/ozone-default.xml | 8 +
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 21 +++
.../common/report/PipelineReportPublisher.java | 73 +++++++++
.../common/report/ReportPublisherFactory.java | 4 +
.../states/endpoint/RegisterEndpointTask.java | 8 +-
.../transport/server/XceiverServerGrpc.java | 16 ++
.../transport/server/XceiverServerSpi.java | 9 ++
.../server/ratis/XceiverServerRatis.java | 63 ++++----
.../container/ozoneimpl/OzoneContainer.java | 12 ++
.../StorageContainerDatanodeProtocol.java | 10 +-
.../protocol/StorageContainerNodeProtocol.java | 6 +-
...rDatanodeProtocolClientSideTranslatorPB.java | 6 +-
...rDatanodeProtocolServerSideTranslatorPB.java | 5 +-
.../StorageContainerDatanodeProtocol.proto | 10 ++
.../ozone/container/common/ScmTestMock.java | 8 +-
.../hdds/scm/container/ContainerMapping.java | 19 ---
.../scm/container/ContainerReportHandler.java | 6 +-
.../hadoop/hdds/scm/container/Mapping.java | 15 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 15 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 19 +--
.../hdds/scm/node/states/Node2ContainerMap.java | 123 ++------------
.../hdds/scm/node/states/Node2ObjectsMap.java | 162 +++++++++++++++++++
.../hdds/scm/node/states/ReportResult.java | 105 ++++++------
.../hdds/scm/pipelines/Node2PipelineMap.java | 45 +-----
.../scm/pipelines/PipelineCloseHandler.java | 24 ++-
.../hdds/scm/pipelines/PipelineManager.java | 52 +++---
.../scm/pipelines/PipelineReportHandler.java | 59 +++++++
.../hdds/scm/pipelines/PipelineSelector.java | 103 ++++++------
.../scm/pipelines/ratis/RatisManagerImpl.java | 41 ++---
.../standalone/StandaloneManagerImpl.java | 44 ++---
.../server/SCMDatanodeHeartbeatDispatcher.java | 23 +++
.../scm/server/SCMDatanodeProtocolServer.java | 16 +-
.../scm/server/StorageContainerManager.java | 11 +-
.../org/apache/hadoop/hdds/scm/TestUtils.java | 10 +-
.../hdds/scm/container/MockNodeManager.java | 4 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 6 +-
.../scm/node/states/TestNode2ContainerMap.java | 28 ++--
.../ozone/container/common/TestEndPoint.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 5 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +-
.../hdds/scm/pipeline/TestSCMRestart.java | 20 ++-
.../apache/hadoop/ozone/RatisTestHelper.java | 9 ++
.../hadoop/ozone/web/client/TestKeys.java | 2 +-
.../hadoop/ozone/web/client/TestKeysRatis.java | 2 -
hadoop-project/pom.xml | 2 +-
51 files changed, 809 insertions(+), 476 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 946abfb..4c4de7f 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -110,7 +110,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
callRatisRpc(pipeline.getMachines(), (raftClient, peer) -> raftClient
- .groupRemove(group.getGroupId(), peer.getId()));
+ .groupRemove(group.getGroupId(), true, peer.getId()));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 492be82..856d113 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -46,6 +46,11 @@ public final class HddsConfigKeys {
public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
"60s";
+ public static final String HDDS_PIPELINE_REPORT_INTERVAL =
+ "hdds.pipeline.report.interval";
+ public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
+ "60s";
+
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
"hdds.command.status.report.interval";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
index ef148e5..777efa7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
-import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.List;
/**
@@ -83,7 +83,7 @@ public class Pipeline {
this.type = replicationType;
this.factor = replicationFactor;
this.id = id;
- datanodes = new TreeMap<>();
+ datanodes = new ConcurrentHashMap<>();
}
@Override
@@ -151,9 +151,21 @@ public class Pipeline {
return getDatanodes().get(leaderID);
}
- public void addMember(DatanodeDetails datanodeDetails) {
- datanodes.put(datanodeDetails.getUuid().toString(),
- datanodeDetails);
+ /**
+ * Adds a datanode to pipeline
+ * @param datanodeDetails datanode to be added.
+ * @return true if the dn was not earlier present, false otherwise
+ */
+ public boolean addMember(DatanodeDetails datanodeDetails) {
+ return datanodes.put(datanodeDetails.getUuid().toString(),
+ datanodeDetails) == null;
+
+ }
+
+ public void resetPipeline() {
+ // reset datanodes in pipeline and learn about them through
+ // pipeline reports on SCM restart
+ datanodes.clear();
}
public Map<String, DatanodeDetails> getDatanodes() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
index 473ebc5..6e27a71 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
@@ -28,7 +28,7 @@ import java.util.UUID;
* in Ratis as RaftGroupId, GroupID is used by the datanodes to initialize
* the ratis group they are part of.
*/
-public class PipelineID {
+public final class PipelineID implements Comparable<PipelineID> {
private UUID id;
private RaftGroupId groupId;
@@ -42,8 +42,12 @@ public class PipelineID {
return new PipelineID(UUID.randomUUID());
}
+ public static PipelineID valueOf(UUID id) {
+ return new PipelineID(id);
+ }
+
public static PipelineID valueOf(RaftGroupId groupId) {
- return new PipelineID(groupId.getUuid());
+ return valueOf(groupId.getUuid());
}
public RaftGroupId getRaftGroupID() {
@@ -68,6 +72,11 @@ public class PipelineID {
}
@Override
+ public int compareTo(PipelineID o) {
+ return this.id.compareTo(o.id);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) {
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index a74124e..f7681e8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -224,6 +224,14 @@
received from SCM to SCM. Unit could be defined with postfix
(ns,ms,s,m,h,d)</description>
</property>
+ <property>
+ <name>hdds.pipeline.report.interval</name>
+ <value>60000ms</value>
+ <tag>OZONE, PIPELINE, MANAGEMENT</tag>
+ <description>Time interval of the datanode to send pipeline report. Each
+ datanode periodically send pipeline report to SCM. Unit could be
+ defined with postfix (ns,ms,s,m,h,d)</description>
+ </property>
<!--Ozone Settings-->
<property>
<name>ozone.administrators</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 580d027..d505be3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -18,12 +18,15 @@
package org.apache.hadoop.hdds.scm;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Map;
@@ -312,4 +315,22 @@ public final class HddsServerUtil {
services.put(OZONE_SCM_SERVICE_ID, serviceInstances);
return services;
}
+
+ public static String getOzoneDatanodeRatisDirectory(Configuration conf) {
+ final String ratisDir = File.separator + "ratis";
+ String storageDir = conf.get(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
+
+ if (Strings.isNullOrEmpty(storageDir)) {
+ storageDir = conf.get(OzoneConfigKeys
+ .OZONE_METADATA_DIRS);
+ Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " +
+ "cannot be null, Please check your configs.");
+ storageDir = storageDir.concat(ratisDir);
+ LOG.warn("Storage directory for Ratis is not configured." +
+ "Mapping Ratis storage under {}. It is a good idea " +
+ "to map this to an SSD disk.", storageDir);
+ }
+ return storageDir;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java
new file mode 100644
index 0000000..e7f4347
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.report;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT;
+
+
+/**
+ * Publishes Pipeline which will be sent to SCM as part of heartbeat.
+ * PipelineReport consist of the following information about each containers:
+ * - pipelineID
+ *
+ */
+public class PipelineReportPublisher extends
+ ReportPublisher<PipelineReportsProto> {
+
+ private Long pipelineReportInterval = null;
+
+ @Override
+ protected long getReportFrequency() {
+ if (pipelineReportInterval == null) {
+ pipelineReportInterval = getConf().getTimeDuration(
+ HDDS_PIPELINE_REPORT_INTERVAL,
+ HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+ getConf());
+
+ Preconditions.checkState(
+ heartbeatFrequency <= pipelineReportInterval,
+ HDDS_PIPELINE_REPORT_INTERVAL +
+ " cannot be configured lower than heartbeat frequency.");
+ }
+ // Add a random delay (0~30s) on top of the pipeline report
+ // interval (60s) so tha the SCM is overwhelmed by the pipeline reports
+ // sent in sync.
+ return pipelineReportInterval + getRandomReportDelay();
+ }
+
+ private long getRandomReportDelay() {
+ return RandomUtils.nextLong(0, pipelineReportInterval);
+ }
+
+ @Override
+ protected PipelineReportsProto getReport() {
+ return getContext().getParent().getContainer().getPipelineReport();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
index ea89280..1c456a0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.container.common.report;
import com.google.protobuf.GeneratedMessage;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.proto.
+ StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
@@ -53,6 +55,8 @@ public class ReportPublisherFactory {
ContainerReportPublisher.class);
report2publisher.put(CommandStatusReportsProto.class,
CommandStatusReportPublisher.class);
+ report2publisher.put(PipelineReportsProto.class,
+ PipelineReportPublisher.class);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index ccab095..690aa01 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -21,6 +21,8 @@ import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.ozone.container.common.statemachine
.EndpointStateMachine;
import org.apache.hadoop.hdds.protocol.proto
@@ -108,13 +110,15 @@ public final class RegisterEndpointTask implements
rpcEndPoint.lock();
try {
- ContainerReportsProto contianerReport = datanodeContainerManager
+ ContainerReportsProto containerReport = datanodeContainerManager
.getContainerReport();
NodeReportProto nodeReport = datanodeContainerManager.getNodeReport();
+ PipelineReportsProto pipelineReportsProto =
+ datanodeContainerManager.getPipelineReport();
// TODO : Add responses to the command Queue.
SCMRegisteredResponseProto response = rpcEndPoint.getEndPoint()
.register(datanodeDetails.getProtoBufMessage(), nodeReport,
- contianerReport);
+ containerReport, pipelineReportsProto);
Preconditions.checkState(UUID.fromString(response.getDatanodeUUID())
.equals(datanodeDetails.getUuid()),
"Unexpected datanode ID in the response.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 4a90144..83e742c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -38,6 +41,9 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketAddress;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
/**
* Creates a Grpc server endpoint that acts as the communication layer for
@@ -47,6 +53,7 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
private static final Logger
LOG = LoggerFactory.getLogger(XceiverServerGrpc.class);
private int port;
+ private UUID id;
private Server server;
private final ContainerDispatcher storageContainer;
@@ -59,6 +66,7 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
ContainerDispatcher dispatcher, BindableService... additionalServices) {
Preconditions.checkNotNull(conf);
+ this.id = datanodeDetails.getUuid();
this.port = conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
// Get an available port on current node and
@@ -123,4 +131,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
HddsProtos.PipelineID pipelineID) {
storageContainer.dispatch(request);
}
+
+ @Override
+ public List<PipelineReport> getPipelineReport() {
+ return Collections.singletonList(
+ PipelineReport.newBuilder()
+ .setPipelineID(PipelineID.valueOf(id).getProtobuf())
+ .build());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
index 1863f6d..8c3fa5c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
@@ -21,8 +21,11 @@ package org.apache.hadoop.ozone.container.common.transport.server;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReport;
import java.io.IOException;
+import java.util.List;
/** A server endpoint that acts as the communication layer for Ozone
* containers. */
@@ -49,4 +52,10 @@ public interface XceiverServerSpi {
void submitRequest(ContainerCommandRequestProto request,
HddsProtos.PipelineID pipelineID)
throws IOException;
+
+ /**
+ * Get pipeline report for the XceiverServer instance.
+ * @return list of report for each pipeline.
+ */
+ List<PipelineReport> getPipelineReport();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 24ea0b9..d88995b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -19,17 +19,18 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -68,6 +69,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketAddress;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
@@ -96,12 +99,12 @@ public final class XceiverServerRatis implements XceiverServerSpi {
private final ReplicationLevel replicationLevel;
private long nodeFailureTimeoutMs;
- private XceiverServerRatis(DatanodeDetails dd, int port, String storageDir,
+ private XceiverServerRatis(DatanodeDetails dd, int port,
ContainerDispatcher dispatcher, Configuration conf, StateContext context)
throws IOException {
Objects.requireNonNull(dd, "id == null");
this.port = port;
- RaftProperties serverProperties = newRaftProperties(conf, storageDir);
+ RaftProperties serverProperties = newRaftProperties(conf);
final int numWriteChunkThreads = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT);
@@ -118,15 +121,13 @@ public final class XceiverServerRatis implements XceiverServerSpi {
new ContainerStateMachine(dispatcher, chunkExecutor, this);
this.server = RaftServer.newBuilder()
.setServerId(RatisHelper.toRaftPeerId(dd))
- .setGroup(RatisHelper.emptyRaftGroup())
.setProperties(serverProperties)
.setStateMachine(stateMachine)
.build();
}
- private RaftProperties newRaftProperties(Configuration conf,
- String storageDir) {
+ private RaftProperties newRaftProperties(Configuration conf) {
final RaftProperties properties = new RaftProperties();
// Set rpc type
@@ -235,6 +236,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
nodeFailureTimeoutMs = nodeFailureTimeout.toLong(TimeUnit.MILLISECONDS);
// Set the ratis storage directory
+ String storageDir = HddsServerUtil.getOzoneDatanodeRatisDirectory(conf);
RaftServerConfigKeys.setStorageDir(properties, new File(storageDir));
// For grpc set the maximum message size
@@ -253,23 +255,9 @@ public final class XceiverServerRatis implements XceiverServerSpi {
public static XceiverServerRatis newXceiverServerRatis(
DatanodeDetails datanodeDetails, Configuration ozoneConf,
ContainerDispatcher dispatcher, StateContext context) throws IOException {
- final String ratisDir = File.separator + "ratis";
int localPort = ozoneConf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT);
- String storageDir = ozoneConf.get(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
-
- if (Strings.isNullOrEmpty(storageDir)) {
- storageDir = ozoneConf.get(OzoneConfigKeys
- .OZONE_METADATA_DIRS);
- Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " +
- "cannot be null, Please check your configs.");
- storageDir = storageDir.concat(ratisDir);
- LOG.warn("Storage directory for Ratis is not configured. Mapping Ratis " +
- "storage under {}. It is a good idea to map this to an SSD disk.",
- storageDir);
- }
// Get an available port on current node and
// use that as the container port
@@ -282,13 +270,6 @@ public final class XceiverServerRatis implements XceiverServerSpi {
socket.bind(address);
localPort = socket.getLocalPort();
LOG.info("Found a free port for the server : {}", localPort);
- // If we have random local ports configured this means that it
- // probably running under MiniOzoneCluster. Ratis locks the storage
- // directories, so we need to pass different local directory for each
- // local instance. So we map ratis directories under datanode ID.
- storageDir =
- storageDir.concat(File.separator +
- datanodeDetails.getUuidString());
} catch (IOException e) {
LOG.error("Unable find a random free port for the server, "
+ "fallback to use default port {}", localPort, e);
@@ -296,7 +277,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
}
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.RATIS, localPort));
- return new XceiverServerRatis(datanodeDetails, localPort, storageDir,
+ return new XceiverServerRatis(datanodeDetails, localPort,
dispatcher, ozoneConf, context);
}
@@ -363,7 +344,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
public void submitRequest(
ContainerCommandRequestProto request, HddsProtos.PipelineID pipelineID)
throws IOException {
- // ReplicationLevel.ALL ensures the transactions corresponding to
+ // ReplicationLevel.MAJORITY ensures the transactions corresponding to
// the request here are applied on all the raft servers.
RaftClientRequest raftClientRequest =
createRaftClientRequest(request, pipelineID,
@@ -427,13 +408,27 @@ public final class XceiverServerRatis implements XceiverServerSpi {
+ ".Reason : " + action.getClosePipeline().getDetailedReason());
}
- void handleNodeSlowness(
- RaftGroup group, RoleInfoProto roleInfoProto) {
+ @Override
+ public List<PipelineReport> getPipelineReport() {
+ try {
+ Iterable<RaftGroupId> gids = server.getGroupIds();
+ List<PipelineReport> reports = new ArrayList<>();
+ for (RaftGroupId groupId : gids) {
+ reports.add(PipelineReport.newBuilder()
+ .setPipelineID(PipelineID.valueOf(groupId).getProtobuf())
+ .build());
+ }
+ return reports;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ void handleNodeSlowness(RaftGroup group, RoleInfoProto roleInfoProto) {
handlePipelineFailure(group.getGroupId(), roleInfoProto);
}
- void handleNoLeader(
- RaftGroup group, RoleInfoProto roleInfoProto) {
+ void handleNoLeader(RaftGroup group, RoleInfoProto roleInfoProto) {
handlePipelineFailure(group.getGroupId(), roleInfoProto);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 72a5804..ebacf75 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -164,6 +166,16 @@ public class OzoneContainer {
return this.containerSet.getContainerReport();
}
+ public PipelineReportsProto getPipelineReport() {
+ PipelineReportsProto.Builder pipelineReportsProto =
+ PipelineReportsProto.newBuilder();
+ for (XceiverServerSpi serverInstance : server) {
+ pipelineReportsProto
+ .addAllPipelineReport(serverInstance.getPipelineReport());
+ }
+ return pipelineReportsProto.build();
+ }
+
/**
* Submit ContainerRequest.
* @param request
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index a950a31..9296524 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
@@ -69,9 +71,11 @@ public interface StorageContainerDatanodeProtocol {
* @param containerReportsRequestProto - Container Reports.
* @return SCM Command.
*/
- SCMRegisteredResponseProto register(DatanodeDetailsProto datanodeDetails,
- NodeReportProto nodeReport, ContainerReportsProto
- containerReportsRequestProto) throws IOException;
+ SCMRegisteredResponseProto register(
+ DatanodeDetailsProto datanodeDetails,
+ NodeReportProto nodeReport,
+ ContainerReportsProto containerReportsRequestProto,
+ PipelineReportsProto pipelineReports) throws IOException;
/**
* Used by datanode to send block deletion ACK to SCM.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
index c9ef43f..b3c3eb3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
@@ -51,10 +53,12 @@ public interface StorageContainerNodeProtocol {
* Register the node if the node finds that it is not registered with any SCM.
* @param datanodeDetails DatanodeDetails
* @param nodeReport NodeReportProto
+ * @param pipelineReport PipelineReportsProto
* @return SCMHeartbeatResponseProto
*/
RegisteredCommand register(DatanodeDetails datanodeDetails,
- NodeReportProto nodeReport);
+ NodeReportProto nodeReport,
+ PipelineReportsProto pipelineReport);
/**
* Send heartbeat to indicate the datanode is alive and doing well.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
index 40fe189..b9cf6f9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
@@ -20,6 +20,8 @@ import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
@@ -149,12 +151,14 @@ public class StorageContainerDatanodeProtocolClientSideTranslatorPB
@Override
public SCMRegisteredResponseProto register(
DatanodeDetailsProto datanodeDetailsProto, NodeReportProto nodeReport,
- ContainerReportsProto containerReportsRequestProto)
+ ContainerReportsProto containerReportsRequestProto,
+ PipelineReportsProto pipelineReportsProto)
throws IOException {
SCMRegisterRequestProto.Builder req =
SCMRegisterRequestProto.newBuilder();
req.setDatanodeDetails(datanodeDetailsProto);
req.setContainerReport(containerReportsRequestProto);
+ req.setPipelineReports(pipelineReportsProto);
req.setNodeReport(nodeReport);
final SCMRegisteredResponseProto response;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
index 7e8bd8a..ed01822 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.protocolPB;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMRegisterRequestProto;
@@ -76,8 +78,9 @@ public class StorageContainerDatanodeProtocolServerSideTranslatorPB
ContainerReportsProto containerRequestProto = request
.getContainerReport();
NodeReportProto dnNodeReport = request.getNodeReport();
+ PipelineReportsProto pipelineReport = request.getPipelineReports();
return impl.register(request.getDatanodeDetails(), dnNodeReport,
- containerRequestProto);
+ containerRequestProto, pipelineReport);
} catch (IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 0a69343..78758cb 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -52,6 +52,7 @@ message SCMRegisterRequestProto {
required DatanodeDetailsProto datanodeDetails = 1;
required NodeReportProto nodeReport = 2;
required ContainerReportsProto containerReport = 3;
+ required PipelineReportsProto pipelineReports = 4;
}
/**
@@ -82,6 +83,7 @@ message SCMHeartbeatRequestProto {
optional CommandStatusReportsProto commandStatusReport = 4;
optional ContainerActionsProto containerActions = 5;
optional PipelineActionsProto pipelineActions = 6;
+ optional PipelineReportsProto pipelineReports = 7;
}
/*
@@ -163,6 +165,14 @@ message ContainerAction {
optional Reason reason = 3;
}
+message PipelineReport {
+ required PipelineID pipelineID = 1;
+}
+
+message PipelineReportsProto {
+ repeated PipelineReport pipelineReport = 1;
+}
+
message PipelineActionsProto {
repeated PipelineAction pipelineActions = 1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
index 751775f..27b6272 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
@@ -18,6 +18,10 @@ package org.apache.hadoop.ozone.container.common;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatus;
import org.apache.hadoop.hdds.scm.VersionInfo;
@@ -214,8 +218,8 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
public StorageContainerDatanodeProtocolProtos
.SCMRegisteredResponseProto register(
DatanodeDetailsProto datanodeDetailsProto, NodeReportProto nodeReport,
- StorageContainerDatanodeProtocolProtos.ContainerReportsProto
- containerReportsRequestProto)
+ ContainerReportsProto containerReportsRequestProto,
+ PipelineReportsProto pipelineReportsProto)
throws IOException {
rpcCount.incrementAndGet();
updateNodeReport(datanodeDetailsProto, nodeReport);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 206e24b..eb0a0b4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -466,24 +466,6 @@ public class ContainerMapping implements Mapping {
return new ContainerWithPipeline(containerInfo, pipeline);
}
- public void handlePipelineClose(PipelineID pipelineID) {
- try {
- Pipeline pipeline = pipelineSelector.getPipeline(pipelineID);
- if (pipeline != null) {
- pipelineSelector.finalizePipeline(pipeline);
- } else {
- LOG.debug("pipeline:{} not found", pipelineID);
- }
- } catch (Exception e) {
- LOG.info("failed to close pipeline:{}", pipelineID, e);
- }
- }
-
- public Set<PipelineID> getPipelineOnDatanode(
- DatanodeDetails datanodeDetails) {
- return pipelineSelector.getPipelineId(datanodeDetails.getUuid());
- }
-
/**
* Process container report from Datanode.
* <p>
@@ -710,7 +692,6 @@ public class ContainerMapping implements Mapping {
return containerStore;
}
- @VisibleForTesting
public PipelineSelector getPipelineSelector() {
return pipelineSelector;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index dcbd49c..3f156de 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -89,20 +89,20 @@ public class ContainerReportHandler implements
.map(ContainerID::new)
.collect(Collectors.toSet());
- ReportResult reportResult = node2ContainerMap
+ ReportResult<ContainerID> reportResult = node2ContainerMap
.processReport(datanodeOrigin.getUuid(), containerIds);
//we have the report, so we can update the states for the next iteration.
node2ContainerMap
.setContainersForDatanode(datanodeOrigin.getUuid(), containerIds);
- for (ContainerID containerID : reportResult.getMissingContainers()) {
+ for (ContainerID containerID : reportResult.getMissingEntries()) {
containerStateManager
.removeContainerReplica(containerID, datanodeOrigin);
checkReplicationState(containerID, publisher);
}
- for (ContainerID containerID : reportResult.getNewContainers()) {
+ for (ContainerID containerID : reportResult.getNewEntries()) {
containerStateManager.addContainerReplica(containerID, datanodeOrigin);
checkReplicationState(containerID, publisher);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
index 1b0c57c..5ed80cb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
@@ -25,13 +25,12 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/**
* Mapping class contains the mapping from a name to a pipeline mapping. This is
@@ -138,15 +137,5 @@ public interface Mapping extends Closeable {
String owner, ReplicationType type, ReplicationFactor factor,
LifeCycleState state) throws IOException;
- /**
- * Handle a pipeline close event.
- * @param pipelineID pipeline id
- */
- void handlePipelineClose(PipelineID pipelineID);
-
- /**
- * Get set of pipeline for a specific datanode.
- * @param datanodeDetails datanode for which pipelines needs to be fetched.
- */
- Set<PipelineID> getPipelineOnDatanode(DatanodeDetails datanodeDetails);
+ PipelineSelector getPipelineSelector();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 9d72eb1..745e052 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -27,10 +27,13 @@ import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
.DeleteBlockCommandStatus;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
.ReplicationStatus;
-import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler.CloseContainerRetryableReq;
+import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler
+ .CloseContainerRetryableReq;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.PipelineActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.ContainerActionsFromDatanode;
@@ -72,8 +75,7 @@ public final class SCMEvents {
/**
* ContainerReports are send out by Datanodes. This report is received by
- * SCMDatanodeHeartbeatDispatcher and Container_Report Event
- * isTestSCMDatanodeHeartbeatDispatcher generated.
+ * SCMDatanodeHeartbeatDispatcher and Container_Report Event is generated.
*/
public static final TypedEvent<ContainerReportFromDatanode> CONTAINER_REPORT =
new TypedEvent<>(ContainerReportFromDatanode.class, "Container_Report");
@@ -87,6 +89,13 @@ public final class SCMEvents {
"Container_Actions");
/**
+ * PipelineReports are send out by Datanodes. This report is received by
+ * SCMDatanodeHeartbeatDispatcher and Pipeline_Report Event is generated.
+ */
+ public static final TypedEvent<PipelineReportFromDatanode> PIPELINE_REPORT =
+ new TypedEvent<>(PipelineReportFromDatanode.class, "Pipeline_Report");
+
+ /**
* PipelineActions are sent by Datanode. This event is received by
* SCMDatanodeHeartbeatDispatcher and PIPELINE_ACTIONS event is generated.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index fca08bd..58da1cc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdds.scm.node;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -363,7 +365,8 @@ public class SCMNodeManager
*/
@Override
public RegisteredCommand register(
- DatanodeDetails datanodeDetails, NodeReportProto nodeReport) {
+ DatanodeDetails datanodeDetails, NodeReportProto nodeReport,
+ PipelineReportsProto pipelineReportsProto) {
InetAddress dnAddress = Server.getRemoteIp();
if (dnAddress != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
index b435e77..ddbba82 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
@@ -19,17 +19,13 @@
package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.container.Mapping;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
+import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Set;
-
/**
* Handles Stale node event.
*/
@@ -37,22 +33,17 @@ public class StaleNodeHandler implements EventHandler<DatanodeDetails> {
static final Logger LOG = LoggerFactory.getLogger(StaleNodeHandler.class);
private final Node2ContainerMap node2ContainerMap;
- private final Mapping containerManager;
+ private final PipelineSelector pipelineSelector;
public StaleNodeHandler(Node2ContainerMap node2ContainerMap,
- Mapping containerManager) {
+ PipelineSelector pipelineSelector) {
this.node2ContainerMap = node2ContainerMap;
- this.containerManager = containerManager;
+ this.pipelineSelector = pipelineSelector;
}
@Override
public void onMessage(DatanodeDetails datanodeDetails,
EventPublisher publisher) {
- Set<PipelineID> pipelineIDs =
- containerManager.getPipelineOnDatanode(datanodeDetails);
- for (PipelineID id : pipelineIDs) {
- LOG.info("closing pipeline {}.", id);
- publisher.fireEvent(SCMEvents.PIPELINE_CLOSE, id);
- }
+ pipelineSelector.handleStaleNode(datanodeDetails);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
index 97c254b..549080a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
@@ -18,13 +18,9 @@
package org.apache.hadoop.hdds.scm.node.states;
-import java.util.Collections;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
-import java.util.TreeSet;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -32,34 +28,29 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .DUPLICATE_DATANODE;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
.NO_SUCH_DATANODE;
/**
* This data structure maintains the list of containers that is on a datanode.
* This information is built from the DN container reports.
*/
-public class Node2ContainerMap {
- private final Map<UUID, Set<ContainerID>> dn2ContainerMap;
+public class Node2ContainerMap extends Node2ObjectsMap<ContainerID> {
/**
* Constructs a Node2ContainerMap Object.
*/
public Node2ContainerMap() {
- dn2ContainerMap = new ConcurrentHashMap<>();
+ super();
}
/**
- * Returns true if this a datanode that is already tracked by
- * Node2ContainerMap.
+ * Returns null if there no containers associated with this datanode ID.
*
- * @param datanodeID - UUID of the Datanode.
- * @return True if this is tracked, false if this map does not know about it.
+ * @param datanode - UUID
+ * @return Set of containers or Null.
*/
- public boolean isKnownDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- return dn2ContainerMap.containsKey(datanodeID);
+ public Set<ContainerID> getContainers(UUID datanode) {
+ return getObjects(datanode);
}
/**
@@ -70,13 +61,7 @@ public class Node2ContainerMap {
*/
public void insertNewDatanode(UUID datanodeID, Set<ContainerID> containerIDs)
throws SCMException {
- Preconditions.checkNotNull(containerIDs);
- Preconditions.checkNotNull(datanodeID);
- if (dn2ContainerMap.putIfAbsent(datanodeID, new HashSet<>(containerIDs))
- != null) {
- throw new SCMException("Node already exists in the map",
- DUPLICATE_DATANODE);
- }
+ super.insertNewDatanode(datanodeID, containerIDs);
}
/**
@@ -91,103 +76,15 @@ public class Node2ContainerMap {
Set<ContainerID> containers) throws SCMException {
Preconditions.checkNotNull(datanodeID);
Preconditions.checkNotNull(containers);
- if (dn2ContainerMap
+ if (dn2ObjectMap
.computeIfPresent(datanodeID, (k, v) -> new HashSet<>(containers))
== null) {
throw new SCMException("No such datanode", NO_SUCH_DATANODE);
}
}
- /**
- * Removes datanode Entry from the map.
- *
- * @param datanodeID - Datanode ID.
- */
- public void removeDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- dn2ContainerMap.computeIfPresent(datanodeID, (k, v) -> null);
- }
-
- /**
- * Returns null if there no containers associated with this datanode ID.
- *
- * @param datanode - UUID
- * @return Set of containers or Null.
- */
- public Set<ContainerID> getContainers(UUID datanode) {
- Preconditions.checkNotNull(datanode);
- return dn2ContainerMap.computeIfPresent(datanode, (k, v) ->
- Collections.unmodifiableSet(v));
- }
-
- public ReportResult processReport(UUID datanodeID, Set<ContainerID>
- containers) {
- Preconditions.checkNotNull(datanodeID);
- Preconditions.checkNotNull(containers);
-
- if (!isKnownDatanode(datanodeID)) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.NEW_DATANODE_FOUND)
- .setNewContainers(containers)
- .build();
- }
-
- // Conditions like Zero length containers should be handled by removeAll.
- Set<ContainerID> currentSet = dn2ContainerMap.get(datanodeID);
- TreeSet<ContainerID> newContainers = new TreeSet<>(containers);
- newContainers.removeAll(currentSet);
-
- TreeSet<ContainerID> missingContainers = new TreeSet<>(currentSet);
- missingContainers.removeAll(containers);
-
- if (newContainers.isEmpty() && missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.ALL_IS_WELL)
- .build();
- }
-
- if (newContainers.isEmpty() && !missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.MISSING_CONTAINERS)
- .setMissingContainers(missingContainers)
- .build();
- }
-
- if (!newContainers.isEmpty() && missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.NEW_CONTAINERS_FOUND)
- .setNewContainers(newContainers)
- .build();
- }
-
- if (!newContainers.isEmpty() && !missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.MISSING_AND_NEW_CONTAINERS_FOUND)
- .setNewContainers(newContainers)
- .setMissingContainers(missingContainers)
- .build();
- }
-
- // default status & Make compiler happy
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.ALL_IS_WELL)
- .build();
- }
-
- /**
- * Results possible from processing a container report by
- * Node2ContainerMapper.
- */
- public enum ReportStatus {
- ALL_IS_WELL,
- MISSING_CONTAINERS,
- NEW_CONTAINERS_FOUND,
- MISSING_AND_NEW_CONTAINERS_FOUND,
- NEW_DATANODE_FOUND
- }
-
@VisibleForTesting
public int size() {
- return dn2ContainerMap.size();
+ return dn2ObjectMap.size();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
new file mode 100644
index 0000000..e49a79c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.scm.node.states;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+
+import java.util.UUID;
+import java.util.Set;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.HashSet;
+import java.util.Collections;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.DUPLICATE_DATANODE;
+
+/**
+ * This data structure maintains the list of containers that is on a datanode.
+ * This information is built from the DN container reports.
+ */
+public class Node2ObjectsMap<T> {
+ protected final Map<UUID, Set<T>> dn2ObjectMap;
+
+ /**
+ * Constructs a Node2ContainerMap Object.
+ */
+ public Node2ObjectsMap() {
+ dn2ObjectMap = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * Returns true if this a datanode that is already tracked by
+ * Node2ContainerMap.
+ *
+ * @param datanodeID - UUID of the Datanode.
+ * @return True if this is tracked, false if this map does not know about it.
+ */
+ public boolean isKnownDatanode(UUID datanodeID) {
+ Preconditions.checkNotNull(datanodeID);
+ return dn2ObjectMap.containsKey(datanodeID);
+ }
+
+ /**
+ * Insert a new datanode into Node2Container Map.
+ *
+ * @param datanodeID -- Datanode UUID
+ * @param containerIDs - List of ContainerIDs.
+ */
+ public void insertNewDatanode(UUID datanodeID, Set<T> containerIDs)
+ throws SCMException {
+ Preconditions.checkNotNull(containerIDs);
+ Preconditions.checkNotNull(datanodeID);
+ if (dn2ObjectMap.putIfAbsent(datanodeID, new HashSet<>(containerIDs))
+ != null) {
+ throw new SCMException("Node already exists in the map",
+ DUPLICATE_DATANODE);
+ }
+ }
+
+ /**
+ * Removes datanode Entry from the map.
+ *
+ * @param datanodeID - Datanode ID.
+ */
+ void removeDatanode(UUID datanodeID) {
+ Preconditions.checkNotNull(datanodeID);
+ dn2ObjectMap.computeIfPresent(datanodeID, (k, v) -> null);
+ }
+
+ /**
+ * Returns null if there no containers associated with this datanode ID.
+ *
+ * @param datanode - UUID
+ * @return Set of containers or Null.
+ */
+ Set<T> getObjects(UUID datanode) {
+ Preconditions.checkNotNull(datanode);
+ final Set<T> s = dn2ObjectMap.get(datanode);
+ return s != null? Collections.unmodifiableSet(s): Collections.emptySet();
+ }
+
+ public ReportResult.ReportResultBuilder<T> newBuilder() {
+ return new ReportResult.ReportResultBuilder<>();
+ }
+
+ public ReportResult<T> processReport(UUID datanodeID, Set<T> objects) {
+ Preconditions.checkNotNull(datanodeID);
+ Preconditions.checkNotNull(objects);
+
+ if (!isKnownDatanode(datanodeID)) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.NEW_DATANODE_FOUND)
+ .setNewEntries(objects)
+ .build();
+ }
+
+ // Conditions like Zero length containers should be handled by removeAll.
+ Set<T> currentSet = dn2ObjectMap.get(datanodeID);
+ TreeSet<T> newObjects = new TreeSet<>(objects);
+ newObjects.removeAll(currentSet);
+
+ TreeSet<T> missingObjects = new TreeSet<>(currentSet);
+ missingObjects.removeAll(objects);
+
+ if (newObjects.isEmpty() && missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.ALL_IS_WELL)
+ .build();
+ }
+
+ if (newObjects.isEmpty() && !missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.MISSING_ENTRIES)
+ .setMissingEntries(missingObjects)
+ .build();
+ }
+
+ if (!newObjects.isEmpty() && missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.NEW_ENTRIES_FOUND)
+ .setNewEntries(newObjects)
+ .build();
+ }
+
+ if (!newObjects.isEmpty() && !missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.MISSING_AND_NEW_ENTRIES_FOUND)
+ .setNewEntries(newObjects)
+ .setMissingEntries(missingObjects)
+ .build();
+ }
+
+ // default status & Make compiler happy
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.ALL_IS_WELL)
+ .build();
+ }
+
+ @VisibleForTesting
+ public int size() {
+ return dn2ObjectMap.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
index 9bb6cf1..0c7610f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
@@ -19,83 +19,92 @@
package org.apache.hadoop.hdds.scm.node.states;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-
import java.util.Collections;
import java.util.Set;
import com.google.common.base.Preconditions;
/**
- * A Container Report gets processsed by the Node2Container and returns
- * Report Result class.
+ * A Container/Pipeline Report gets processed by the
+ * Node2Container/Node2Pipeline and returns Report Result class.
*/
-public class ReportResult {
- private Node2ContainerMap.ReportStatus status;
- private Set<ContainerID> missingContainers;
- private Set<ContainerID> newContainers;
-
- ReportResult(Node2ContainerMap.ReportStatus status,
- Set<ContainerID> missingContainers,
- Set<ContainerID> newContainers) {
+public final class ReportResult<T> {
+ private ReportStatus status;
+ private Set<T> missingEntries;
+ private Set<T> newEntries;
+
+ private ReportResult(ReportStatus status,
+ Set<T> missingEntries,
+ Set<T> newEntries) {
this.status = status;
- Preconditions.checkNotNull(missingContainers);
- Preconditions.checkNotNull(newContainers);
- this.missingContainers = missingContainers;
- this.newContainers = newContainers;
+ Preconditions.checkNotNull(missingEntries);
+ Preconditions.checkNotNull(newEntries);
+ this.missingEntries = missingEntries;
+ this.newEntries = newEntries;
}
- public Node2ContainerMap.ReportStatus getStatus() {
+ public ReportStatus getStatus() {
return status;
}
- public Set<ContainerID> getMissingContainers() {
- return missingContainers;
+ public Set<T> getMissingEntries() {
+ return missingEntries;
}
- public Set<ContainerID> getNewContainers() {
- return newContainers;
+ public Set<T> getNewEntries() {
+ return newEntries;
}
- static class ReportResultBuilder {
- private Node2ContainerMap.ReportStatus status;
- private Set<ContainerID> missingContainers;
- private Set<ContainerID> newContainers;
-
- static ReportResultBuilder newBuilder() {
- return new ReportResultBuilder();
- }
-
- public ReportResultBuilder setStatus(
- Node2ContainerMap.ReportStatus newstatus) {
- this.status = newstatus;
+ /**
+ * Result after processing report for node2Object map.
+ * @param <T>
+ */
+ public static class ReportResultBuilder<T> {
+ private ReportStatus status;
+ private Set<T> missingEntries;
+ private Set<T> newEntries;
+
+ public ReportResultBuilder<T> setStatus(
+ ReportStatus newStatus) {
+ this.status = newStatus;
return this;
}
- public ReportResultBuilder setMissingContainers(
- Set<ContainerID> missingContainersLit) {
- this.missingContainers = missingContainersLit;
+ public ReportResultBuilder<T> setMissingEntries(
+ Set<T> missingEntriesList) {
+ this.missingEntries = missingEntriesList;
return this;
}
- public ReportResultBuilder setNewContainers(
- Set<ContainerID> newContainersList) {
- this.newContainers = newContainersList;
+ public ReportResultBuilder<T> setNewEntries(
+ Set<T> newEntriesList) {
+ this.newEntries = newEntriesList;
return this;
}
- ReportResult build() {
+ public ReportResult<T> build() {
- Set<ContainerID> nullSafeMissingContainers = this.missingContainers;
- Set<ContainerID> nullSafeNewContainers = this.newContainers;
- if (nullSafeNewContainers == null) {
- nullSafeNewContainers = Collections.emptySet();
+ Set<T> nullSafeMissingEntries = this.missingEntries;
+ Set<T> nullSafeNewEntries = this.newEntries;
+ if (nullSafeNewEntries == null) {
+ nullSafeNewEntries = Collections.emptySet();
}
- if (nullSafeMissingContainers == null) {
- nullSafeMissingContainers = Collections.emptySet();
+ if (nullSafeMissingEntries == null) {
+ nullSafeMissingEntries = Collections.emptySet();
}
- return new ReportResult(status, nullSafeMissingContainers,
- nullSafeNewContainers);
+ return new ReportResult<T>(status, nullSafeMissingEntries,
+ nullSafeNewEntries);
}
}
+
+ /**
+ * Results possible from processing a report.
+ */
+ public enum ReportStatus {
+ ALL_IS_WELL,
+ MISSING_ENTRIES,
+ NEW_ENTRIES_FOUND,
+ MISSING_AND_NEW_ENTRIES_FOUND,
+ NEW_DATANODE_FOUND,
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
index 363ce71..87f2222 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
@@ -16,19 +16,15 @@
*
*/
-package org.apache.hadoop.hdds.scm.pipelines;
+package org.apache.hadoop.hdds.scm.node.states;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import java.util.Collections;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
/**
* This data structure maintains the list of pipelines which the given datanode is a part of. This
@@ -36,33 +32,11 @@ import java.util.concurrent.ConcurrentHashMap;
*
* <p>TODO: this information needs to be regenerated from pipeline reports on SCM restart
*/
-public class Node2PipelineMap {
- private final Map<UUID, Set<PipelineID>> dn2PipelineMap;
+public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
/** Constructs a Node2PipelineMap Object. */
public Node2PipelineMap() {
- dn2PipelineMap = new ConcurrentHashMap<>();
- }
-
- /**
- * Returns true if this a datanode that is already tracked by Node2PipelineMap.
- *
- * @param datanodeID - UUID of the Datanode.
- * @return True if this is tracked, false if this map does not know about it.
- */
- private boolean isKnownDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- return dn2PipelineMap.containsKey(datanodeID);
- }
-
- /**
- * Removes datanode Entry from the map.
- *
- * @param datanodeID - Datanode ID.
- */
- public synchronized void removeDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- dn2PipelineMap.computeIfPresent(datanodeID, (k, v) -> null);
+ super();
}
/**
@@ -72,9 +46,7 @@ public class Node2PipelineMap {
* @return Set of pipelines or Null.
*/
public Set<PipelineID> getPipelines(UUID datanode) {
- Preconditions.checkNotNull(datanode);
- final Set<PipelineID> s = dn2PipelineMap.get(datanode);
- return s != null? Collections.unmodifiableSet(s): Collections.emptySet();
+ return getObjects(datanode);
}
/**
@@ -85,7 +57,7 @@ public class Node2PipelineMap {
public synchronized void addPipeline(Pipeline pipeline) {
for (DatanodeDetails details : pipeline.getDatanodes().values()) {
UUID dnId = details.getUuid();
- dn2PipelineMap.computeIfAbsent(dnId, k -> new HashSet<>())
+ dn2ObjectMap.computeIfAbsent(dnId, k -> new HashSet<>())
.add(pipeline.getId());
}
}
@@ -93,16 +65,11 @@ public class Node2PipelineMap {
public synchronized void removePipeline(Pipeline pipeline) {
for (DatanodeDetails details : pipeline.getDatanodes().values()) {
UUID dnId = details.getUuid();
- dn2PipelineMap.computeIfPresent(
- dnId,
+ dn2ObjectMap.computeIfPresent(dnId,
(k, v) -> {
v.remove(pipeline.getId());
return v;
});
}
}
-
- public Map<UUID, Set<PipelineID>> getDn2PipelineMap() {
- return Collections.unmodifiableMap(dn2PipelineMap);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
index 733dec5..e49678f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
@@ -17,22 +17,36 @@
package org.apache.hadoop.hdds.scm.pipelines;
-import org.apache.hadoop.hdds.scm.container.Mapping;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Handles pipeline close event.
*/
public class PipelineCloseHandler implements EventHandler<PipelineID> {
- private final Mapping mapping;
- public PipelineCloseHandler(Mapping mapping) {
- this.mapping = mapping;
+ private static final Logger LOG = LoggerFactory
+ .getLogger(PipelineCloseHandler.class);
+
+ private final PipelineSelector pipelineSelector;
+ public PipelineCloseHandler(PipelineSelector pipelineSelector) {
+ this.pipelineSelector = pipelineSelector;
}
@Override
public void onMessage(PipelineID pipelineID, EventPublisher publisher) {
- mapping.handlePipelineClose(pipelineID);
+ Pipeline pipeline = pipelineSelector.getPipeline(pipelineID);
+ try {
+ if (pipeline != null) {
+ pipelineSelector.finalizePipeline(pipeline);
+ } else {
+ LOG.debug("pipeline:{} not found", pipelineID);
+ }
+ } catch (Exception e) {
+ LOG.info("failed to close pipeline:{}", pipelineID, e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
index 07ff2b0..ca2e878 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
@@ -18,6 +18,8 @@ package org.apache.hadoop.hdds.scm.pipelines;
import java.util.ArrayList;
import java.util.LinkedList;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -36,7 +38,7 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class PipelineManager {
private static final Logger LOG =
LoggerFactory.getLogger(PipelineManager.class);
- private final ArrayList<ActivePipelines> activePipelines;
+ protected final ArrayList<ActivePipelines> activePipelines;
public PipelineManager() {
activePipelines = new ArrayList<>();
@@ -45,7 +47,10 @@ public abstract class PipelineManager {
}
}
- private static class ActivePipelines {
+ /**
+ * List of active pipelines.
+ */
+ public static class ActivePipelines {
private final List<PipelineID> activePipelines;
private final AtomicInteger pipelineIndex;
@@ -55,10 +60,12 @@ public abstract class PipelineManager {
}
void addPipeline(PipelineID pipelineID) {
- activePipelines.add(pipelineID);
+ if (!activePipelines.contains(pipelineID)) {
+ activePipelines.add(pipelineID);
+ }
}
- void removePipeline(PipelineID pipelineID) {
+ public void removePipeline(PipelineID pipelineID) {
activePipelines.remove(pipelineID);
}
@@ -117,17 +124,6 @@ public abstract class PipelineManager {
.addPipeline(pipeline.getId());
}
- protected static int getReplicationCount(ReplicationFactor factor) {
- switch (factor) {
- case ONE:
- return 1;
- case THREE:
- return 3;
- default:
- throw new IllegalArgumentException("Unexpected replication count");
- }
- }
-
public abstract Pipeline allocatePipeline(
ReplicationFactor replicationFactor);
@@ -137,6 +133,14 @@ public abstract class PipelineManager {
*/
public abstract void initializePipeline(Pipeline pipeline) throws IOException;
+ public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
+ if (pipeline.addMember(dn)
+ &&(pipeline.getDatanodes().size() == pipeline.getFactor().getNumber())
+ && pipeline.getLifeCycleState() == HddsProtos.LifeCycleState.OPEN) {
+ addOpenPipeline(pipeline);
+ }
+ }
+
/**
* Creates a pipeline with a specified replication factor and type.
* @param replicationFactor - Replication Factor.
@@ -157,27 +161,11 @@ public abstract class PipelineManager {
* Remove the pipeline from active allocation.
* @param pipeline pipeline to be finalized
*/
- public synchronized void finalizePipeline(Pipeline pipeline) {
- activePipelines.get(pipeline.getFactor().ordinal())
- .removePipeline(pipeline.getId());
- }
+ public abstract boolean finalizePipeline(Pipeline pipeline);
/**
*
* @param pipeline
*/
public abstract void closePipeline(Pipeline pipeline) throws IOException;
-
- /**
- * list members in the pipeline.
- * @return the datanode
- */
- public abstract List<DatanodeDetails> getMembers(PipelineID pipelineID)
- throws IOException;
-
- /**
- * Update the datanode list of the pipeline.
- */
- public abstract void updatePipeline(PipelineID pipelineID,
- List<DatanodeDetails> newDatanodes) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java
new file mode 100644
index 0000000..933792b
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipelines;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.server
+ .SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles Node Reports from datanode.
+ */
+public class PipelineReportHandler implements
+ EventHandler<PipelineReportFromDatanode> {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(PipelineReportHandler.class);
+ private final PipelineSelector pipelineSelector;
+
+ public PipelineReportHandler(PipelineSelector pipelineSelector) {
+ Preconditions.checkNotNull(pipelineSelector);
+ this.pipelineSelector = pipelineSelector;
+ }
+
+ @Override
+ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode,
+ EventPublisher publisher) {
+ Preconditions.checkNotNull(pipelineReportFromDatanode);
+ DatanodeDetails dn = pipelineReportFromDatanode.getDatanodeDetails();
+ PipelineReportsProto pipelineReport =
+ pipelineReportFromDatanode.getReport();
+ Preconditions.checkNotNull(dn, "Pipeline Report is "
+ + "missing DatanodeDetails.");
+ LOGGER.trace("Processing pipeline report for dn: {}", dn);
+ pipelineSelector.processPipelineReport(dn, pipelineReport);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HADOOP-15755.
StringUtils#createStartupShutdownMessage throws NPE when args is null.
Contributed by Lokesh Jain and Dinesh Chitlangia
Posted by xk...@apache.org.
HADOOP-15755. StringUtils#createStartupShutdownMessage throws NPE when args is null. Contributed by Lokesh Jain and Dinesh Chitlangia
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e71f61ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e71f61ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e71f61ec
Branch: refs/heads/HDFS-12943
Commit: e71f61ecb87e04727a5a76e578a75714c9db6706
Parents: 5896372
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 15:55:09 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 15:57:33 2018 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/util/StringUtils.java | 2 +-
.../test/java/org/apache/hadoop/util/TestStringUtils.java | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e71f61ec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 3db805f..f49698c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -743,7 +743,7 @@ public class StringUtils {
return toStartupShutdownString("STARTUP_MSG: ", new String[] {
"Starting " + classname,
" host = " + hostname,
- " args = " + Arrays.asList(args),
+ " args = " + (args != null ? Arrays.asList(args) : new ArrayList<>()),
" version = " + VersionInfo.getVersion(),
" classpath = " + System.getProperty("java.class.path"),
" build = " + VersionInfo.getUrl() + " -r "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e71f61ec/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
index 3fdc1bb..f05b589 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
@@ -503,6 +503,15 @@ public class TestStringUtils extends UnitTestcaseTimeLimit {
escapedStr, StringUtils.escapeHTML(htmlStr));
}
+ @Test
+ public void testCreateStartupShutdownMessage() {
+ //pass null args and method must still return a string beginning with
+ // "STARTUP_MSG"
+ String msg = StringUtils.createStartupShutdownMessage(
+ this.getClass().getName(), "test.host", null);
+ assertTrue(msg.startsWith("STARTUP_MSG:"));
+ }
+
// Benchmark for StringUtils split
public static void main(String []args) {
final String TO_SPLIT = "foo,bar,baz,blah,blah";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDDS-464. Fix
TestCloseContainerHandlingByClient. Contributed by Lokesh Jain.
Posted by xk...@apache.org.
HDDS-464. Fix TestCloseContainerHandlingByClient.
Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f938925b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f938925b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f938925b
Branch: refs/heads/HDFS-12943
Commit: f938925bde1d481bacb2546096a1e49fe796b411
Parents: 6ff509c
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Sep 18 10:44:56 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Sep 18 10:45:51 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientRatis.java | 4 ++
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 +++++++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 14 +++++++
.../main/java/org/apache/ratis/RatisHelper.java | 40 +++++++++++++++++---
.../common/src/main/resources/ozone-default.xml | 19 ++++++++++
.../server/ratis/XceiverServerRatis.java | 13 +++++++
.../rpc/TestCloseContainerHandlingByClient.java | 5 +--
7 files changed, 100 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index f0db7b5..946abfb 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -208,6 +208,10 @@ public final class XceiverClientRatis extends XceiverClientSpi {
public ContainerCommandResponseProto sendCommand(
ContainerCommandRequestProto request) throws IOException {
final RaftClientReply reply = sendRequest(request);
+ if (reply == null) {
+ throw new IOException(
+ String.format("Could not execute the request %s", request));
+ }
Preconditions.checkState(reply.isSuccess());
return ContainerCommandResponseProto.parseFrom(
reply.getMessage().getContent());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 5b25779..63f5916 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -75,6 +75,19 @@ public final class ScmConfigKeys {
public static final TimeDuration
DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
+ public static final String DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY =
+ "dfs.ratis.client.request.max.retries";
+ public static final int DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT = 180;
+ public static final String DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY =
+ "dfs.ratis.client.request.retry.interval";
+ public static final TimeDuration
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT =
+ TimeDuration.valueOf(100, TimeUnit.MILLISECONDS);
+ public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
+ "dfs.ratis.server.retry-cache.timeout.duration";
+ public static final TimeDuration
+ DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
+ TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.server.request.timeout.duration";
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 54ec139..599b4e8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -237,6 +237,20 @@ public final class OzoneConfigKeys {
public static final TimeDuration
DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT =
ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT;
+ public static final String DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY;
+ public static final int DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT;
+ public static final String DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY;
+ public static final TimeDuration
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT;
+ public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
+ ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY;
+ public static final TimeDuration
+ DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
+ ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT;
public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY;
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index d851992..04bfeb2 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -34,6 +34,7 @@ import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.shaded.com.google.protobuf.ByteString;
import org.apache.ratis.shaded.proto.RaftProtos;
+import org.apache.ratis.util.Preconditions;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
@@ -48,6 +49,9 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY;
+
/**
* Ratis helper methods.
*/
@@ -162,12 +166,38 @@ public interface RatisHelper {
static RetryPolicy createRetryPolicy(Configuration conf) {
int maxRetryCount =
- conf.getInt(OzoneConfigKeys.OZONE_CLIENT_MAX_RETRIES, OzoneConfigKeys.
- OZONE_CLIENT_MAX_RETRIES_DEFAULT);
+ conf.getInt(OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY,
+ OzoneConfigKeys.
+ DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT);
long retryInterval = conf.getTimeDuration(OzoneConfigKeys.
- OZONE_CLIENT_RETRY_INTERVAL, OzoneConfigKeys.
- OZONE_CLIENT_RETRY_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS.MILLISECONDS);
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY, OzoneConfigKeys.
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ long leaderElectionTimeout = conf.getTimeDuration(
+ DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
+ DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ long clientRequestTimeout = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY,
+ OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ long retryCacheTimeout = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY,
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ Preconditions
+ .assertTrue(maxRetryCount * retryInterval > 5 * leaderElectionTimeout,
+ "Please make sure dfs.ratis.client.request.max.retries * "
+ + "dfs.ratis.client.request.retry.interval > "
+ + "5 * dfs.ratis.leader.election.minimum.timeout.duration");
+ Preconditions.assertTrue(
+ maxRetryCount * (retryInterval + clientRequestTimeout)
+ < retryCacheTimeout,
+ "Please make sure "
+ + "(dfs.ratis.client.request.max.retries * "
+ + "(dfs.ratis.client.request.retry.interval + "
+ + "dfs.ratis.client.request.timeout.duration)) "
+ + "< dfs.ratis.server.retry-cache.timeout.duration");
TimeDuration sleepDuration =
TimeDuration.valueOf(retryInterval, TimeUnit.MILLISECONDS);
RetryPolicy retryPolicy = RetryPolicies
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index e160f25..a74124e 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -158,6 +158,25 @@
<description>The timeout duration for ratis client request.</description>
</property>
<property>
+ <name>dfs.ratis.client.request.max.retries</name>
+ <value>180</value>
+ <tag>OZONE, RATIS, MANAGEMENT</tag>
+ <description>Number of retries for ratis client request.</description>
+ </property>
+ <property>
+ <name>dfs.ratis.client.request.retry.interval</name>
+ <value>100ms</value>
+ <tag>OZONE, RATIS, MANAGEMENT</tag>
+ <description>Interval between successive retries for a ratis client request.
+ </description>
+ </property>
+ <property>
+ <name>dfs.ratis.server.retry-cache.timeout.duration</name>
+ <value>600000ms</value>
+ <tag>OZONE, RATIS, MANAGEMENT</tag>
+ <description>Retry Cache entry timeout for ratis server.</description>
+ </property>
+ <property>
<name>dfs.ratis.server.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index a57997d..24ea0b9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -182,6 +182,19 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftServerConfigKeys.Rpc
.setRequestTimeout(properties, serverRequestTimeout);
+ // set timeout for a retry cache entry
+ timeUnit =
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
+ .getUnit();
+ duration = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY,
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
+ .getDuration(), timeUnit);
+ final TimeDuration retryCacheTimeout =
+ TimeDuration.valueOf(duration, timeUnit);
+ RaftServerConfigKeys.RetryCache
+ .setExpiryTime(properties, retryCacheTimeout);
+
// Set the ratis leader election timeout
TimeUnit leaderElectionMinTimeoutUnit =
OzoneConfigKeys.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index cf38982..83421b2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -55,7 +55,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import java.util.Random;
/**
* Tests Close Container Exception handling by Ozone Client.
@@ -83,9 +82,9 @@ public class TestCloseContainerHandlingByClient {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- // generate a no between 1 to 10
- maxRetries = new Random().nextInt(10);
+ maxRetries = 100;
conf.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_RETRIES, maxRetries);
+ conf.set(OzoneConfigKeys.OZONE_CLIENT_RETRY_INTERVAL, "200ms");
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDDS-502. Exception in OM startup
when running unit tests. Contributed by Arpit Agarwal.
Posted by xk...@apache.org.
HDDS-502. Exception in OM startup when running unit tests. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15ed74fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15ed74fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15ed74fa
Branch: refs/heads/HDFS-12943
Commit: 15ed74fa244eacbe7a8eac877ea64d7a53a1bf9c
Parents: c0956ee
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 19:46:25 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 19:46:25 2018 +0530
----------------------------------------------------------------------
.../scm/server/StorageContainerManager.java | 55 ++++++++++++++++----
.../apache/hadoop/ozone/om/OzoneManager.java | 33 ++++++++++--
2 files changed, 73 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15ed74fa/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2169149..86c061b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -374,7 +374,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
hParser.printGenericCommandUsage(System.err);
System.exit(1);
}
- StorageContainerManager scm = createSCM(hParser.getRemainingArgs(), conf);
+ StorageContainerManager scm = createSCM(
+ hParser.getRemainingArgs(), conf, true);
if (scm != null) {
scm.start();
scm.join();
@@ -389,9 +390,37 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
out.println(USAGE + "\n");
}
- public static StorageContainerManager createSCM(String[] args,
- OzoneConfiguration conf)
- throws IOException {
+ /**
+ * Create an SCM instance based on the supplied command-line arguments.
+ *
+ * This method is intended for unit tests only. It suppresses the
+ * startup/shutdown message and skips registering Unix signal
+ * handlers.
+ *
+ * @param args command line arguments.
+ * @param conf HDDS configuration
+ * @return SCM instance
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public static StorageContainerManager createSCM(
+ String[] args, OzoneConfiguration conf) throws IOException {
+ return createSCM(args, conf, false);
+ }
+
+ /**
+ * Create an SCM instance based on the supplied command-line arguments.
+ *
+ * @param args command-line arguments.
+ * @param conf HDDS configuration
+ * @param printBanner if true, then log a verbose startup message.
+ * @return SCM instance
+ * @throws IOException
+ */
+ private static StorageContainerManager createSCM(
+ String[] args,
+ OzoneConfiguration conf,
+ boolean printBanner) throws IOException {
String[] argv = (args == null) ? new String[0] : args;
if (!HddsUtils.isHddsEnabled(conf)) {
System.err.println(
@@ -407,13 +436,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
switch (startOpt) {
case INIT:
- StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
- LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
+ }
terminate(scmInit(conf) ? 0 : 1);
return null;
case GENCLUSTERID:
- StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
- LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
+ }
System.out.println("Generating new cluster id:");
System.out.println(StorageInfo.newClusterID());
terminate(0);
@@ -423,8 +456,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
terminate(0);
return null;
default:
- StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
- LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
+ }
return new StorageContainerManager(conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15ed74fa/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 71cc6ba..6ea0fe7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -258,7 +258,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
hParser.printGenericCommandUsage(System.err);
System.exit(1);
}
- OzoneManager om = createOm(hParser.getRemainingArgs(), conf);
+ OzoneManager om = createOm(hParser.getRemainingArgs(), conf, true);
if (om != null) {
om.start();
om.join();
@@ -276,14 +276,33 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
/**
* Constructs OM instance based on command line arguments.
*
+ * This method is intended for unit tests only. It suppresses the
+ * startup/shutdown message and skips registering Unix signal
+ * handlers.
+ *
* @param argv Command line arguments
* @param conf OzoneConfiguration
* @return OM instance
* @throws IOException in case OM instance creation fails.
*/
+ @VisibleForTesting
+ public static OzoneManager createOm(
+ String[] argv, OzoneConfiguration conf) throws IOException {
+ return createOm(argv, conf, false);
+ }
- public static OzoneManager createOm(String[] argv,
- OzoneConfiguration conf) throws IOException {
+
+ /**
+ * Constructs OM instance based on command line arguments.
+ *
+ * @param argv Command line arguments
+ * @param conf OzoneConfiguration
+ * @param printBanner if true then log a verbose startup message.
+ * @return OM instance
+ * @throws IOException in case OM instance creation fails.
+ */
+ private static OzoneManager createOm(String[] argv,
+ OzoneConfiguration conf, boolean printBanner) throws IOException {
if (!isHddsEnabled(conf)) {
System.err.println("OM cannot be started in secure mode or when " +
OZONE_ENABLED + " is set to false");
@@ -297,7 +316,9 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
switch (startOpt) {
case CREATEOBJECTSTORE:
- StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ }
terminate(omInit(conf) ? 0 : 1);
return null;
case HELP:
@@ -308,7 +329,9 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
if (argv == null) {
argv = new String[]{};
}
- StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ }
return new OzoneManager(conf);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-8787. Fix broken list items
in PlacementConstraints documentation. Contributed by Masahiro Tanaka.
Posted by xk...@apache.org.
YARN-8787. Fix broken list items in PlacementConstraints documentation. Contributed by Masahiro Tanaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78a0d173
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78a0d173
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78a0d173
Branch: refs/heads/HDFS-12943
Commit: 78a0d173e4f0c2f2679a04edd62a60fb76dde4f0
Parents: b6ad84e
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Sep 18 17:19:33 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Sep 18 17:19:39 2018 +0800
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78a0d173/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
index 4ac1683..a583493 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
@@ -87,6 +87,7 @@ An example of PlacementSpec is the following:
zk=3,NOTIN,NODE,zk:hbase=5,IN,RACK,zk:spark=7,CARDINALITY,NODE,hbase,1,3
```
The above encodes three constraints:
+
* place 3 containers with tag "zk" (standing for ZooKeeper) with node anti-affinity to each other, i.e., do not place more than one container per node (notice that in this first constraint, the SourceTag and the TargetTag of the constraint coincide);
* place 5 containers with tag "hbase" with affinity to a rack on which containers with tag "zk" are running (i.e., an "hbase" container should not be placed at a rack where an "zk" container is running, given that "zk" is the TargetTag of the second constraint);
* place 7 containers with tag "spark" in nodes that have at least one, but no more than three, containers with tag "hbase".
@@ -132,6 +133,7 @@ The example constraints used above could be extended with namespaces as follows:
zk=3,NOTIN,NODE,not-self/zk:hbase=5,IN,RACK,all/zk:spark=7,CARDINALITY,NODE,app-id/appID_0023/hbase,1,3
```
The semantics of these constraints are the following:
+
* place 3 containers with tag "zk" (standing for ZooKeeper) to nodes that do not have "zk" containers from other applications running;
* place 5 containers with tag "hbase" with affinity to a rack on which containers with tag "zk" (from any application, be it the same or a different one) are running;
* place 7 containers with tag "spark" in nodes that have at least one, but no more than three, containers with tag "hbase" belonging to application with ID `appID_0023`.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDDS-500. TestErrorCode.java has
wrong package name. Contributed by Anu Engineer.
Posted by xk...@apache.org.
HDDS-500. TestErrorCode.java has wrong package name. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ff00f55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ff00f55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ff00f55
Branch: refs/heads/HDFS-12943
Commit: 7ff00f558713f2f0755193b7d57ebdad8d8f349a
Parents: f176e8a3
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 15:52:00 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 15:52:00 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/web/client/TestKeys.java | 1 +
.../org/apache/hadoop/ozone/TestErrorCode.java | 53 --------------------
.../apache/hadoop/ozone/web/TestErrorCode.java | 53 ++++++++++++++++++++
3 files changed, 54 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff00f55/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 21887be..6377f11 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -328,6 +328,7 @@ public class TestKeys {
cluster.restartHddsDatanode(datanodeIdx);
}
+ @Ignore("Causes a JVm exit")
@Test
public void testPutAndGetKeyWithDnRestart() throws Exception {
runTestPutAndGetKeyWithDnRestart(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff00f55/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java b/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java
deleted file mode 100644
index abb61bb..0000000
--- a/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertEquals;
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.getRequestID;
-
-/**
- * Test Ozone Error Codes.
- */
-public class TestErrorCode {
- /**
- * Test Error Generator functions.
- */
- @Test
- public void testErrorGen() {
- OzoneException e = ErrorTable
- .newError(ErrorTable.ACCESS_DENIED, getRequestID(), "/test/path",
- "localhost");
- assertEquals(e.getHostID(), "localhost");
- assertEquals(e.getShortMessage(),
- ErrorTable.ACCESS_DENIED.getShortMessage());
- }
-
- @Test
- public void testErrorGenWithException() {
- OzoneException e =
- new OzoneException(ErrorTable.ACCESS_DENIED.getHttpCode(),
- "short message", new Exception("Hello"));
- assertEquals("short message", e.getShortMessage());
- assertEquals("Hello", e.getMessage());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff00f55/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java b/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java
new file mode 100644
index 0000000..abb61bb
--- /dev/null
+++ b/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.web;
+
+import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.apache.hadoop.ozone.web.utils.OzoneUtils.getRequestID;
+
+/**
+ * Test Ozone Error Codes.
+ */
+public class TestErrorCode {
+ /**
+ * Test Error Generator functions.
+ */
+ @Test
+ public void testErrorGen() {
+ OzoneException e = ErrorTable
+ .newError(ErrorTable.ACCESS_DENIED, getRequestID(), "/test/path",
+ "localhost");
+ assertEquals(e.getHostID(), "localhost");
+ assertEquals(e.getShortMessage(),
+ ErrorTable.ACCESS_DENIED.getShortMessage());
+ }
+
+ @Test
+ public void testErrorGenWithException() {
+ OzoneException e =
+ new OzoneException(ErrorTable.ACCESS_DENIED.getHttpCode(),
+ "short message", new Exception("Hello"));
+ assertEquals("short message", e.getShortMessage());
+ assertEquals("Hello", e.getMessage());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-503. Suppress
ShellBasedUnixGroupsMapping exception in tests. Contributed by Arpit Agarwal.
Posted by xk...@apache.org.
HDDS-503. Suppress ShellBasedUnixGroupsMapping exception in tests. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17f5651a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17f5651a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17f5651a
Branch: refs/heads/HDFS-12943
Commit: 17f5651a5124c6d00fc990f252de7af5c226a314
Parents: 7ff00f5
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 16:06:21 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 16:06:21 2018 -0700
----------------------------------------------------------------------
.../integration-test/src/test/resources/log4j.properties | 5 ++++-
hadoop-ozone/ozonefs/src/test/resources/log4j.properties | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17f5651a/hadoop-ozone/integration-test/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/resources/log4j.properties b/hadoop-ozone/integration-test/src/test/resources/log4j.properties
index cad9dd1..9ec5a92 100644
--- a/hadoop-ozone/integration-test/src/test/resources/log4j.properties
+++ b/hadoop-ozone/integration-test/src/test/resources/log4j.properties
@@ -15,4 +15,7 @@ log4j.rootLogger=info,stdout
log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
\ No newline at end of file
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17f5651a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
index 3bf1619..a7b5aa9 100644
--- a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
+++ b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
@@ -17,6 +17,7 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
# for debugging low level Ozone operations, uncomment this line
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-8791. Trim docker inspect
output for line feed for STOPSIGNAL parsing. Contributed by Chandni Singh
Posted by xk...@apache.org.
YARN-8791. Trim docker inspect output for line feed for STOPSIGNAL parsing.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efdea85a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efdea85a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efdea85a
Branch: refs/heads/HDFS-12943
Commit: efdea85ad1cd4cc5a2a306898dbdb2c14b952d02
Parents: 1824d5d
Author: Eric Yang <ey...@apache.org>
Authored: Wed Sep 19 13:16:11 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Sep 19 13:16:11 2018 -0400
----------------------------------------------------------------------
.../linux/runtime/DockerLinuxContainerRuntime.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efdea85a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 2b53f13..55dfdc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1282,7 +1282,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
DockerInspectCommand.STATUS_TEMPLATE,
DockerInspectCommand.STOPSIGNAL_TEMPLATE}, delimiter);
try {
- String output = executeDockerInspect(containerId, inspectCommand);
+ String output = executeDockerInspect(containerId, inspectCommand).trim();
if (!output.isEmpty()) {
String[] statusAndSignal = StringUtils.split(output, delimiter);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDDS-483. Update ozone
Documentation to fix below issues. Contributed by Namit Maheshwari.
Posted by xk...@apache.org.
HDDS-483. Update ozone Documentation to fix below issues.
Contributed by Namit Maheshwari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48319d6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48319d6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48319d6e
Branch: refs/heads/HDFS-12943
Commit: 48319d6eee6ecea9992f3959ed75c892aa09f66c
Parents: e4fca6a
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Sep 18 10:23:36 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Sep 18 10:23:36 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/JavaApi.md | 4 ++++
hadoop-ozone/docs/content/OzoneFS.md | 6 +++---
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48319d6e/hadoop-ozone/docs/content/JavaApi.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/JavaApi.md b/hadoop-ozone/docs/content/JavaApi.md
index 589122c..1d32bed 100644
--- a/hadoop-ozone/docs/content/JavaApi.md
+++ b/hadoop-ozone/docs/content/JavaApi.md
@@ -129,8 +129,12 @@ introStream.close();
Here is a complete example of the code that we just wrote. Please note the close functions being called in this program.
{{< highlight java >}}
+// Let us create a client
OzoneClient ozClient = OzoneClientFactory.getClient();
+// Get a reference to the ObjectStore using the client
+ObjectStore objectStore = ozClient.getObjectStore();
+
// Let us create a volume to store our game assets.
// This default arguments for creating that volume.
objectStore.createVolume(“assets”);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48319d6e/hadoop-ozone/docs/content/OzoneFS.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md
index 07961d9..d0621be 100644
--- a/hadoop-ozone/docs/content/OzoneFS.md
+++ b/hadoop-ozone/docs/content/OzoneFS.md
@@ -32,8 +32,8 @@ To create an ozone file system, we have to choose a bucket where the file system
Please run the following commands to create a volume and bucket, if you don't have them already.
{{< highlight bash >}}
-ozone oz volume create /volume
-ozone oz bucket create /volume/bucket
+ozone sh volume create /volume
+ozone sh bucket create /volume/bucket
{{< /highlight >}}
Once this is created, please make sure that bucket exists via the listVolume or listBucket commands.
@@ -56,7 +56,7 @@ This will make this bucket to be the default file system for HDFS dfs commands a
You also need to add the ozone-filesystem.jar file to the classpath:
{{< highlight bash >}}
-export HADOOP_CLASSPATH=/opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+export HADOOP_CLASSPATH=/opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar:$HADOOP_CLASSPATH
{{< /highlight >}}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: MAPREDUCE-7137.
MRAppBenchmark.benchmark1() fails with NullPointerException. Contributed by
Oleksandr Shevchenko
Posted by xk...@apache.org.
MAPREDUCE-7137. MRAppBenchmark.benchmark1() fails with NullPointerException. Contributed by Oleksandr Shevchenko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34b2237e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34b2237e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34b2237e
Branch: refs/heads/HDFS-12943
Commit: 34b2237e420cfbe3a97ddd44968de8bbe1ed30ab
Parents: 5c2ae7e
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 16:56:31 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 16:56:31 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b2237e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
index 025a8fa..5e6697b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -184,7 +185,7 @@ public class MRAppBenchmark {
}
}
- @Test
+ @Test(timeout = 60000)
public void benchmark1() throws Exception {
int maps = 100; // Adjust for benchmarking. Start with thousands.
int reduces = 0;
@@ -211,6 +212,7 @@ public class MRAppBenchmark {
Records.newRecord(RegisterApplicationMasterResponse.class);
response.setMaximumResourceCapability(Resource.newInstance(
10240, 1));
+ response.setQueue("queue1");
return response;
}
@@ -252,6 +254,7 @@ public class MRAppBenchmark {
response.setAllocatedContainers(containers);
response.setResponseId(request.getResponseId() + 1);
response.setNumClusterNodes(350);
+ response.setApplicationPriority(Priority.newInstance(100));
return response;
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDDS-481. Classes are missing from
the shaded ozonefs jar. Contributed by Bharat Viswanadham.
Posted by xk...@apache.org.
HDDS-481. Classes are missing from the shaded ozonefs jar. Contributed by Bharat Viswanadham.
(cherry picked from commit e043783917ed58a1a967ca67b7240b1b3e846a69)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/001611cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/001611cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/001611cc
Branch: refs/heads/HDFS-12943
Commit: 001611cca3b2f7c78dace73234850934b73d3ef3
Parents: bbeca01
Author: Márton Elek <el...@apache.org>
Authored: Tue Sep 18 10:26:33 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Tue Sep 18 10:32:36 2018 +0200
----------------------------------------------------------------------
hadoop-ozone/ozonefs/pom.xml | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/001611cc/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index d55cda5..8174b74 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -50,20 +50,20 @@
<configuration>
<artifactSet>
<includes>
- <include>com.google.guava:guava:jar</include>
- <include>org.slf4j:slf4j-api:jar</include>
+ <include>com.google.guava:guava</include>
+ <include>org.slf4j:slf4j-api</include>
<include>com.google.protobuf:protobuf-java</include>
- <include>com.nimbusds:nimbus-jose-jwt:jar</include>
+ <include>com.nimbusds:nimbus-jose-jwt</include>
<include>com.github.stephenc.jcip:jcip-annotations</include>
- <include>com.google.code.findbugs:jsr305:jar</include>
+ <include>com.google.code.findbugs:jsr305</include>
<include>org.apache.hadoop:hadoop-ozone-client</include>
<include>org.apache.hadoop:hadoop-hdds-client</include>
<include>org.apache.hadoop:hadoop-hdds-common</include>
<include>org.fusesource.leveldbjni:leveldbjni-all</include>
<include>org.apache.ratis:ratis-server</include>
- <include>org.apache.ratis:ratis-proto-shaded:jar</include>
+ <include>org.apache.ratis:ratis-proto-shaded</include>
<include>com.google.auto.value:auto-value-annotations</include>
- <include>com.squareup:javapoet:jar</include>
+ <include>com.squareup:javapoet</include>
<include>org.jctools:jctools-core</include>
<include>org.apache.ratis:ratis-common</include>
<include>org.apache.ratis:ratis-client</include>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HDFS-13749. [SBN read] Use
getServiceStatus to discover observer namenodes. Contributed by Chao Sun.
Posted by xk...@apache.org.
HDFS-13749. [SBN read] Use getServiceStatus to discover observer namenodes. Contributed by Chao Sun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77e106f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77e106f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77e106f7
Branch: refs/heads/HDFS-12943
Commit: 77e106f74715a7bb79effe38c245ec7c1566be72
Parents: a3810f7
Author: Erik Krogen <xk...@apache.org>
Authored: Thu Sep 20 13:27:58 2018 -0700
Committer: Erik Krogen <xk...@apache.org>
Committed: Thu Sep 20 13:27:58 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/NameNodeProxiesClient.java | 47 ++++++++-
.../ha/AbstractNNFailoverProxyProvider.java | 36 +++++--
.../namenode/ha/IPFailoverProxyProvider.java | 2 +-
.../namenode/ha/ObserverReadProxyProvider.java | 49 +--------
.../ha/TestObserverReadProxyProvider.java | 105 ++++++++++++++-----
5 files changed, 151 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77e106f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
index 284e4ef..f90d671 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
@@ -25,12 +25,16 @@ import java.net.InetSocketAddress;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.protocolPB.HAServiceProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.namenode.ha.ClientHAProxyFactory;
import org.apache.hadoop.hdfs.server.namenode.ha.HAProxyFactory;
import org.apache.hadoop.ipc.AlignmentContext;
+import org.apache.hadoop.ipc.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,13 +66,14 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
/**
- * Create proxy objects with {@link ClientProtocol} to communicate with a remote
- * NN. Generally use {@link NameNodeProxiesClient#createProxyWithClientProtocol(
+ * Create proxy objects with {@link ClientProtocol} and
+ * {@link HAServiceProtocol} to communicate with a remote NN. For the former,
+ * generally use {@link NameNodeProxiesClient#createProxyWithClientProtocol(
* Configuration, URI, AtomicBoolean)}, which will create either an HA- or
* non-HA-enabled client proxy as appropriate.
*
- * For creating proxy objects with other protocols, please see
- * {@link NameNodeProxies#createProxy(Configuration, URI, Class)}.
+ * For creating proxy objects with other protocols, please see the server-side
+ * counterpart {@code NameNodeProxies#createProxy}
*/
@InterfaceAudience.Private
public class NameNodeProxiesClient {
@@ -76,6 +81,11 @@ public class NameNodeProxiesClient {
private static final Logger LOG = LoggerFactory.getLogger(
NameNodeProxiesClient.class);
+ /** Maximum # of retries for HAProxy with HAServiceProtocol. */
+ private static final int MAX_RETRIES = 3;
+ /** Initial retry delay for HAProxy with HAServiceProtocol. */
+ private static final int DELAY_MILLISECONDS = 200;
+
/**
* Wrapper for a client proxy as well as its associated service ID.
* This is simply used as a tuple-like return type for created NN proxy.
@@ -119,7 +129,6 @@ public class NameNodeProxiesClient {
* @return an object containing both the proxy and the associated
* delegation token service it corresponds to
* @throws IOException if there is an error creating the proxy
- * @see {@link NameNodeProxies#createProxy(Configuration, URI, Class)}.
*/
public static ProxyAndInfo<ClientProtocol> createProxyWithClientProtocol(
Configuration conf, URI nameNodeUri, AtomicBoolean fallbackToSimpleAuth)
@@ -343,6 +352,34 @@ public class NameNodeProxiesClient {
fallbackToSimpleAuth, null);
}
+ /**
+ * Creates a non-HA proxy object with {@link HAServiceProtocol} to the
+ * given NameNode address, using the provided configuration. The proxy will
+ * use the RPC timeout configuration specified via {@link
+ * org.apache.hadoop.fs.CommonConfigurationKeys#IPC_CLIENT_RPC_TIMEOUT_KEY}.
+ * Upon failures, this will retry up to certain times with {@link RetryProxy}.
+ *
+ * @param address the NameNode address
+ * @param conf the configuration to be used
+ * @return a non-HA proxy with {@link HAServiceProtocol}.
+ */
+ public static HAServiceProtocol createNonHAProxyWithHAServiceProtocol(
+ InetSocketAddress address, Configuration conf) throws IOException {
+ RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
+ MAX_RETRIES, DELAY_MILLISECONDS, TimeUnit.MILLISECONDS);
+
+ HAServiceProtocol proxy =
+ new HAServiceProtocolClientSideTranslatorPB(
+ address, conf, NetUtils.getDefaultSocketFactory(conf),
+ Client.getRpcTimeout(conf));
+ return (HAServiceProtocol) RetryProxy.create(
+ HAServiceProtocol.class,
+ new DefaultFailoverProxyProvider<>(HAServiceProtocol.class, proxy),
+ new HashMap<>(),
+ timeoutPolicy
+ );
+ }
+
public static ClientProtocol createProxyWithAlignmentContext(
InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
boolean withRetries, AtomicBoolean fallbackToSimpleAuth,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77e106f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
index 32edb36..1b5ad16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java
@@ -28,11 +28,14 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtilClient;
+import org.apache.hadoop.hdfs.NameNodeProxiesClient;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.io.retry.FailoverProxyProvider;
import org.apache.hadoop.security.UserGroupInformation;
@@ -119,23 +122,44 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
*/
private HAServiceState cachedState;
- public NNProxyInfo(InetSocketAddress address) {
+ /** Proxy for getting HA service status from the given NameNode. */
+ private HAServiceProtocol serviceProxy;
+
+ public NNProxyInfo(InetSocketAddress address, Configuration conf) {
super(null, address.toString());
this.address = address;
+ try {
+ serviceProxy = NameNodeProxiesClient
+ .createNonHAProxyWithHAServiceProtocol(address, conf);
+ } catch (IOException ioe) {
+ LOG.error("Failed to create HAServiceProtocol proxy to NameNode" +
+ " at {}", address, ioe);
+ throw new RuntimeException(ioe);
+ }
}
public InetSocketAddress getAddress() {
return address;
}
- public void setCachedState(HAServiceState state) {
- cachedState = state;
+ public void refreshCachedState() {
+ try {
+ cachedState = serviceProxy.getServiceStatus().getState();
+ } catch (IOException e) {
+ LOG.warn("Failed to connect to {}. Setting cached state to Standby",
+ address, e);
+ cachedState = HAServiceState.STANDBY;
+ }
}
public HAServiceState getCachedState() {
return cachedState;
}
+ @VisibleForTesting
+ public void setServiceProxyForTesting(HAServiceProtocol proxy) {
+ this.serviceProxy = proxy;
+ }
}
@Override
@@ -153,8 +177,8 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
pi.proxy = factory.createProxy(conf,
pi.getAddress(), xface, ugi, false, getFallbackToSimpleAuth());
} catch (IOException ioe) {
- LOG.error("{} Failed to create RPC proxy to NameNode",
- this.getClass().getSimpleName(), ioe);
+ LOG.error("{} Failed to create RPC proxy to NameNode at {}",
+ this.getClass().getSimpleName(), pi.address, ioe);
throw new RuntimeException(ioe);
}
}
@@ -178,7 +202,7 @@ public abstract class AbstractNNFailoverProxyProvider<T> implements
Collection<InetSocketAddress> addressesOfNns = addressesInNN.values();
for (InetSocketAddress address : addressesOfNns) {
- proxies.add(new NNProxyInfo<T>(address));
+ proxies.add(new NNProxyInfo<T>(address, conf));
}
// Randomize the list to prevent all clients pointing to the same one
boolean randomized = getRandomOrder(conf, uri);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77e106f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java
index e703740..8062e79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/IPFailoverProxyProvider.java
@@ -48,7 +48,7 @@ public class IPFailoverProxyProvider<T> extends
public IPFailoverProxyProvider(Configuration conf, URI uri,
Class<T> xface, HAProxyFactory<T> factory) {
super(conf, uri, xface, factory);
- this.nnProxyInfo = new NNProxyInfo<T>(DFSUtilClient.getNNAddress(uri));
+ this.nnProxyInfo = new NNProxyInfo<>(DFSUtilClient.getNNAddress(uri), conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77e106f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java
index e819282..690ee0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java
@@ -27,12 +27,10 @@ import java.net.URI;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.ClientGSIContext;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.io.retry.RetryPolicies;
@@ -40,8 +38,6 @@ import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
import org.apache.hadoop.ipc.AlignmentContext;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.StandbyException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -181,49 +177,6 @@ public class ObserverReadProxyProvider<T extends ClientProtocol>
return lastProxy;
}
- private static <T extends ClientProtocol> HAServiceState getServiceState(
- NNProxyInfo<T> pi) {
- // TODO: should introduce new ClientProtocol method to verify the
- // underlying service state, which does not require superuser access
- // The is a workaround
- IOException ioe = null;
- try {
- // Verify write access first
- pi.proxy.reportBadBlocks(new LocatedBlock[0]);
- return HAServiceState.ACTIVE; // Only active NameNode allows write
- } catch (RemoteException re) {
- IOException sbe = re.unwrapRemoteException(StandbyException.class);
- if (!(sbe instanceof StandbyException)) {
- ioe = re;
- }
- } catch (IOException e) {
- ioe = e;
- }
- if (ioe != null) {
- LOG.warn("Failed to connect to {}", pi.getAddress(), ioe);
- return HAServiceState.STANDBY; // Just assume standby in this case
- // Anything besides observer is fine
- }
- // Verify read access
- // For now we assume only Observer nodes allow reads
- // Stale reads on StandbyNode should be turned off
- try {
- pi.proxy.checkAccess("/", FsAction.READ);
- return HAServiceState.OBSERVER;
- } catch (RemoteException re) {
- IOException sbe = re.unwrapRemoteException(StandbyException.class);
- if (!(sbe instanceof StandbyException)) {
- ioe = re;
- }
- } catch (IOException e) {
- ioe = e;
- }
- if (ioe != null) {
- LOG.warn("Failed to connect to {}", pi.getAddress(), ioe);
- }
- return HAServiceState.STANDBY;
- }
-
/**
* Return the currently used proxy. If there is none, first calls
* {@link #changeProxy(NNProxyInfo)} to initialize one.
@@ -254,7 +207,7 @@ public class ObserverReadProxyProvider<T extends ClientProtocol>
currentProxy = null;
currentIndex = (currentIndex + 1) % nameNodeProxies.size();
currentProxy = createProxyIfNeeded(nameNodeProxies.get(currentIndex));
- currentProxy.setCachedState(getServiceState(currentProxy));
+ currentProxy.refreshCachedState();
LOG.debug("Changed current proxy from {} to {}",
initial == null ? "none" : initial.proxyInfo,
currentProxy.proxyInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77e106f7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java
index 4d5bc13..3f56c96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverReadProxyProvider.java
@@ -22,10 +22,13 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -38,10 +41,12 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import static org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
-
+import static org.mockito.Mockito.when;
/**
* Tests for {@link ObserverReadProxyProvider} under various configurations of
@@ -56,7 +61,7 @@ public class TestObserverReadProxyProvider {
private Configuration conf;
private ObserverReadProxyProvider<ClientProtocol> proxyProvider;
- private ClientProtocolAnswer[] namenodeAnswers;
+ private NameNodeAnswer[] namenodeAnswers;
private String[] namenodeAddrs;
@Before
@@ -70,32 +75,53 @@ public class TestObserverReadProxyProvider {
private void setupProxyProvider(int namenodeCount) throws Exception {
String[] namenodeIDs = new String[namenodeCount];
namenodeAddrs = new String[namenodeCount];
- namenodeAnswers = new ClientProtocolAnswer[namenodeCount];
+ namenodeAnswers = new NameNodeAnswer[namenodeCount];
ClientProtocol[] proxies = new ClientProtocol[namenodeCount];
Map<String, ClientProtocol> proxyMap = new HashMap<>();
+ HAServiceProtocol[] serviceProxies = new HAServiceProtocol[namenodeCount];
+ Map<String, HAServiceProtocol> serviceProxyMap = new HashMap<>();
for (int i = 0; i < namenodeCount; i++) {
namenodeIDs[i] = "nn" + i;
namenodeAddrs[i] = "namenode" + i + ".test:8020";
conf.set(HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns +
"." + namenodeIDs[i], namenodeAddrs[i]);
- namenodeAnswers[i] = new ClientProtocolAnswer();
+ namenodeAnswers[i] = new NameNodeAnswer();
proxies[i] = mock(ClientProtocol.class);
- doWrite(Mockito.doAnswer(namenodeAnswers[i]).when(proxies[i]));
- doRead(Mockito.doAnswer(namenodeAnswers[i]).when(proxies[i]));
+ doWrite(Mockito.doAnswer(namenodeAnswers[i].clientAnswer)
+ .when(proxies[i]));
+ doRead(Mockito.doAnswer(namenodeAnswers[i].clientAnswer)
+ .when(proxies[i]));
+ serviceProxies[i] = mock(HAServiceProtocol.class);
+ Mockito.doAnswer(namenodeAnswers[i].serviceAnswer)
+ .when(serviceProxies[i]).getServiceStatus();
proxyMap.put(namenodeAddrs[i], proxies[i]);
+ serviceProxyMap.put(namenodeAddrs[i], serviceProxies[i]);
}
conf.set(HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns,
Joiner.on(",").join(namenodeIDs));
- proxyProvider = new ObserverReadProxyProvider<>(conf, nnURI,
- ClientProtocol.class, new ClientHAProxyFactory<ClientProtocol>() {
+ proxyProvider = new ObserverReadProxyProvider<ClientProtocol>(conf, nnURI,
+ ClientProtocol.class,
+ new ClientHAProxyFactory<ClientProtocol>() {
+ @Override
+ public ClientProtocol createProxy(Configuration config,
+ InetSocketAddress nnAddr, Class<ClientProtocol> xface,
+ UserGroupInformation ugi, boolean withRetries,
+ AtomicBoolean fallbackToSimpleAuth) {
+ return proxyMap.get(nnAddr.toString());
+ }
+ }) {
@Override
- public ClientProtocol createProxy(Configuration conf,
- InetSocketAddress nnAddr, Class<ClientProtocol> xface,
- UserGroupInformation ugi, boolean withRetries,
- AtomicBoolean fallbackToSimpleAuth) {
- return proxyMap.get(nnAddr.toString());
+ protected List<NNProxyInfo<ClientProtocol>> getProxyAddresses(
+ URI uri, String addressKey) {
+ List<NNProxyInfo<ClientProtocol>> nnProxies =
+ super.getProxyAddresses(uri, addressKey);
+ for (NNProxyInfo<ClientProtocol> nnProxy : nnProxies) {
+ String addressStr = nnProxy.getAddress().toString();
+ nnProxy.setServiceProxyForTesting(serviceProxyMap.get(addressStr));
+ }
+ return nnProxies;
}
- });
+ };
proxyProvider.setObserverReadEnabled(true);
}
@@ -275,39 +301,62 @@ public class TestObserverReadProxyProvider {
}
/**
- * An {@link Answer} used for mocking of a {@link ClientProtocol}. Setting
- * the state or unreachability of this Answer will make the linked
- * ClientProtocol respond as if it was communicating with a NameNode of
- * the corresponding state. It is in Standby state by default.
+ * An {@link Answer} used for mocking of {@link ClientProtocol} and
+ * {@link HAServiceProtocol}. Setting the state or unreachability of this
+ * Answer will make the linked ClientProtocol respond as if it was
+ * communicating with a NameNode of the corresponding state. It is in Standby
+ * state by default.
*/
- private static class ClientProtocolAnswer implements Answer<Void> {
+ private static class NameNodeAnswer {
private volatile boolean unreachable = false;
// Standby state by default
private volatile boolean allowWrites = false;
private volatile boolean allowReads = false;
- @Override
- public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
- if (unreachable) {
- throw new IOException("Unavailable");
+ private ClientProtocolAnswer clientAnswer = new ClientProtocolAnswer();
+ private HAServiceProtocolAnswer serviceAnswer =
+ new HAServiceProtocolAnswer();
+
+ private class HAServiceProtocolAnswer implements Answer<HAServiceStatus> {
+ @Override
+ public HAServiceStatus answer(InvocationOnMock invocation)
+ throws Throwable {
+ HAServiceStatus status = mock(HAServiceStatus.class);
+ if (allowReads && allowWrites) {
+ when(status.getState()).thenReturn(HAServiceState.ACTIVE);
+ } else if (allowReads) {
+ when(status.getState()).thenReturn(HAServiceState.OBSERVER);
+ } else {
+ when(status.getState()).thenReturn(HAServiceState.STANDBY);
+ }
+ return status;
}
- switch (invocationOnMock.getMethod().getName()) {
+ }
+
+ private class ClientProtocolAnswer implements Answer<Void> {
+ @Override
+ public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
+ if (unreachable) {
+ throw new IOException("Unavailable");
+ }
+ switch (invocationOnMock.getMethod().getName()) {
case "reportBadBlocks":
if (!allowWrites) {
- throw new RemoteException(StandbyException.class.getCanonicalName(),
- "No writes!");
+ throw new RemoteException(
+ StandbyException.class.getCanonicalName(), "No writes!");
}
return null;
case "checkAccess":
if (!allowReads) {
- throw new RemoteException(StandbyException.class.getCanonicalName(),
- "No reads!");
+ throw new RemoteException(
+ StandbyException.class.getCanonicalName(), "No reads!");
}
return null;
default:
throw new IllegalArgumentException(
"Only reportBadBlocks and checkAccess supported!");
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDDS-460. Replication manager
failed to import container data. Contributed by Elek, Marton.
Posted by xk...@apache.org.
HDDS-460. Replication manager failed to import container data. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042bf74d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042bf74d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042bf74d
Branch: refs/heads/HDFS-12943
Commit: 042bf74d5eb3dc627658b8c4c027628569169513
Parents: efdea85
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 23:51:50 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 23:51:50 2018 +0530
----------------------------------------------------------------------
.../statemachine/DatanodeStateMachine.java | 19 ++-
.../ReplicateContainerCommandHandler.java | 120 ++------------
.../replication/ContainerReplicator.java | 27 +++
.../DownloadAndImportReplicator.java | 136 ++++++++++++++++
.../replication/GrpcReplicationClient.java | 2 +-
.../replication/ReplicationSupervisor.java | 142 ++++++++++++++++
.../container/replication/ReplicationTask.java | 102 ++++++++++++
.../TestReplicateContainerCommandHandler.java | 163 -------------------
.../replication/TestReplicationSupervisor.java | 143 ++++++++++++++++
.../container/replication/package-info.java | 22 +++
10 files changed, 602 insertions(+), 274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 875d063..1bade8e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -40,7 +40,12 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.DeleteBlocksCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.ReplicateContainerCommandHandler;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.container.replication.ContainerReplicator;
+import org.apache.hadoop.ozone.container.replication.DownloadAndImportReplicator;
+import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
+import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
@@ -69,6 +74,7 @@ public class DatanodeStateMachine implements Closeable {
private AtomicLong nextHB;
private Thread stateMachineThread = null;
private Thread cmdProcessThread = null;
+ private final ReplicationSupervisor supervisor;
/**
* Constructs a a datanode state machine.
@@ -89,14 +95,21 @@ public class DatanodeStateMachine implements Closeable {
new OzoneConfiguration(conf), context);
nextHB = new AtomicLong(Time.monotonicNow());
+ ContainerReplicator replicator =
+ new DownloadAndImportReplicator(container.getContainerSet(),
+ container.getDispatcher(),
+ new SimpleContainerDownloader(conf), new TarContainerPacker());
+
+ supervisor =
+ new ReplicationSupervisor(container.getContainerSet(), replicator, 10);
+
// When we add new handlers just adding a new handler here should do the
// trick.
commandDispatcher = CommandDispatcher.newBuilder()
.addHandler(new CloseContainerCommandHandler())
.addHandler(new DeleteBlocksCommandHandler(container.getContainerSet(),
conf))
- .addHandler(new ReplicateContainerCommandHandler(conf,
- container.getContainerSet(), container.getDispatcher()))
+ .addHandler(new ReplicateContainerCommandHandler(conf, supervisor))
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
@@ -295,6 +308,7 @@ public class DatanodeStateMachine implements Closeable {
public void startDaemon() {
Runnable startStateMachineTask = () -> {
try {
+ supervisor.start();
start();
LOG.info("Ozone container server started.");
} catch (Exception ex) {
@@ -323,6 +337,7 @@ public class DatanodeStateMachine implements Closeable {
*/
public synchronized void stopDaemon() {
try {
+ supervisor.stop();
context.setState(DatanodeStates.SHUTDOWN);
reportManager.shutdown();
this.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
index cb677c2..09c379f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -16,33 +16,17 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-import java.io.FileInputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.statemachine
- .SCMConnectionManager;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.container.replication.ContainerDownloader;
-import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
+import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -58,39 +42,19 @@ public class ReplicateContainerCommandHandler implements CommandHandler {
static final Logger LOG =
LoggerFactory.getLogger(ReplicateContainerCommandHandler.class);
- private ContainerDispatcher containerDispatcher;
-
private int invocationCount;
private long totalTime;
- private ContainerDownloader downloader;
-
private Configuration conf;
- private TarContainerPacker packer = new TarContainerPacker();
-
- private ContainerSet containerSet;
-
- private Lock lock = new ReentrantLock();
+ private ReplicationSupervisor supervisor;
public ReplicateContainerCommandHandler(
Configuration conf,
- ContainerSet containerSet,
- ContainerDispatcher containerDispatcher,
- ContainerDownloader downloader) {
+ ReplicationSupervisor supervisor) {
this.conf = conf;
- this.containerSet = containerSet;
- this.downloader = downloader;
- this.containerDispatcher = containerDispatcher;
- }
-
- public ReplicateContainerCommandHandler(
- Configuration conf,
- ContainerSet containerSet,
- ContainerDispatcher containerDispatcher) {
- this(conf, containerSet, containerDispatcher,
- new SimpleContainerDownloader(conf));
+ this.supervisor = supervisor;
}
@Override
@@ -108,72 +72,12 @@ public class ReplicateContainerCommandHandler implements CommandHandler {
String.format("Replication command is received for container %d "
+ "but the size of source datanodes was 0.", containerID));
- LOG.info("Starting replication of container {} from {}", containerID,
- sourceDatanodes);
- CompletableFuture<Path> tempTarFile = downloader
- .getContainerDataFromReplicas(containerID,
- sourceDatanodes);
-
- CompletableFuture<Void> result =
- tempTarFile.thenAccept(path -> {
- LOG.info("Container {} is downloaded, starting to import.",
- containerID);
- importContainer(containerID, path);
- });
-
- result.whenComplete((aVoid, throwable) -> {
- if (throwable != null) {
- LOG.error("Container replication was unsuccessful .", throwable);
- } else {
- LOG.info("Container {} is replicated successfully", containerID);
- }
- });
- } finally {
- updateCommandStatus(context, command, true, LOG);
+ ReplicationTask replicationTask =
+ new ReplicationTask(containerID, sourceDatanodes);
+ supervisor.addTask(replicationTask);
- }
- }
-
- protected void importContainer(long containerID, Path tarFilePath) {
- lock.lock();
- try {
- ContainerData originalContainerData;
- try (FileInputStream tempContainerTarStream = new FileInputStream(
- tarFilePath.toFile())) {
- byte[] containerDescriptorYaml =
- packer.unpackContainerDescriptor(tempContainerTarStream);
- originalContainerData = ContainerDataYaml.readContainer(
- containerDescriptorYaml);
- }
-
- try (FileInputStream tempContainerTarStream = new FileInputStream(
- tarFilePath.toFile())) {
-
- Handler handler = containerDispatcher.getHandler(
- originalContainerData.getContainerType());
-
- Container container = handler.importContainer(containerID,
- originalContainerData.getMaxSize(),
- tempContainerTarStream,
- packer);
-
- containerSet.addContainer(container);
- }
-
- } catch (Exception e) {
- LOG.error(
- "Can't import the downloaded container data id=" + containerID,
- e);
- try {
- Files.delete(tarFilePath);
- } catch (Exception ex) {
- LOG.error(
- "Container import is failed and the downloaded file can't be "
- + "deleted: "
- + tarFilePath.toAbsolutePath().toString());
- }
} finally {
- lock.unlock();
+ updateCommandStatus(context, command, true, LOG);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
new file mode 100644
index 0000000..827b9d6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+/**
+ * Service to do the real replication task.
+ *
+ * An implementation should download the container and im
+ */
+public interface ContainerReplicator {
+ void replicate(ReplicationTask task);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
new file mode 100644
index 0000000..5ef5841
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask.Status;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Default replication implementation.
+ * <p>
+ * This class does the real job. Executes the download and import the container
+ * to the container set.
+ */
+public class DownloadAndImportReplicator implements ContainerReplicator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DownloadAndImportReplicator.class);
+
+ private final ContainerSet containerSet;
+
+ private final ContainerDispatcher containerDispatcher;
+
+ private final ContainerDownloader downloader;
+
+ private final TarContainerPacker packer;
+
+ public DownloadAndImportReplicator(
+ ContainerSet containerSet,
+ ContainerDispatcher containerDispatcher,
+ ContainerDownloader downloader,
+ TarContainerPacker packer) {
+ this.containerSet = containerSet;
+ this.containerDispatcher = containerDispatcher;
+ this.downloader = downloader;
+ this.packer = packer;
+ }
+
+ public void importContainer(long containerID, Path tarFilePath) {
+ try {
+ ContainerData originalContainerData;
+ try (FileInputStream tempContainerTarStream = new FileInputStream(
+ tarFilePath.toFile())) {
+ byte[] containerDescriptorYaml =
+ packer.unpackContainerDescriptor(tempContainerTarStream);
+ originalContainerData = ContainerDataYaml.readContainer(
+ containerDescriptorYaml);
+ }
+
+ try (FileInputStream tempContainerTarStream = new FileInputStream(
+ tarFilePath.toFile())) {
+
+ Handler handler = containerDispatcher.getHandler(
+ originalContainerData.getContainerType());
+
+ Container container = handler.importContainer(containerID,
+ originalContainerData.getMaxSize(),
+ tempContainerTarStream,
+ packer);
+
+ containerSet.addContainer(container);
+ }
+
+ } catch (Exception e) {
+ LOG.error(
+ "Can't import the downloaded container data id=" + containerID,
+ e);
+ try {
+ Files.delete(tarFilePath);
+ } catch (Exception ex) {
+ LOG.error(
+ "Container import is failed and the downloaded file can't be "
+ + "deleted: "
+ + tarFilePath.toAbsolutePath().toString());
+ }
+ }
+ }
+
+ @Override
+ public void replicate(ReplicationTask task) {
+ long containerID = task.getContainerId();
+
+ List<DatanodeDetails> sourceDatanodes = task.getSources();
+
+ LOG.info("Starting replication of container {} from {}", containerID,
+ sourceDatanodes);
+
+ CompletableFuture<Path> tempTarFile = downloader
+ .getContainerDataFromReplicas(containerID,
+ sourceDatanodes);
+
+ try {
+ //wait for the download. This thread pool is limiting the paralell
+ //downloads, so it's ok to block here and wait for the full download.
+ Path path = tempTarFile.get();
+ LOG.info("Container {} is downloaded, starting to import.",
+ containerID);
+ importContainer(containerID, path);
+ LOG.info("Container {} is replicated successfully", containerID);
+ task.setStatus(Status.DONE);
+ } catch (Exception e) {
+ LOG.error("Container replication was unsuccessful .", e);
+ task.setStatus(Status.FAILED);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 91d098f..3aafb0c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -157,8 +157,8 @@ public class GrpcReplicationClient {
public void onCompleted() {
try {
stream.close();
- response.complete(outputPath);
LOG.info("Container is downloaded to {}", outputPath);
+ response.complete(outputPath);
} catch (IOException e) {
response.completeExceptionally(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
new file mode 100644
index 0000000..1d8d5f6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask.Status;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Single point to schedule the downloading tasks based on priorities.
+ */
+public class ReplicationSupervisor {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReplicationSupervisor.class);
+
+ private final Set<Worker> threadPool = new HashSet<>();
+
+ private final Map<Long, ReplicationTask> queue = new TreeMap();
+
+ private final ContainerSet containerSet;
+
+ private final ContainerReplicator replicator;
+
+ private final int poolSize;
+
+ public ReplicationSupervisor(
+ ContainerSet containerSet,
+ ContainerReplicator replicator, int poolSize) {
+ this.containerSet = containerSet;
+ this.replicator = replicator;
+ this.poolSize = poolSize;
+ }
+
+ public synchronized void addTask(ReplicationTask task) {
+ queue.putIfAbsent(task.getContainerId(), task);
+ synchronized (threadPool) {
+ threadPool.notify();
+ }
+ }
+
+ public void start() {
+ for (int i = 0; i < poolSize; i++) {
+ Worker worker = new Worker();
+ Thread thread = new Thread(worker, "ContainerReplication-" + i);
+ thread.setDaemon(true);
+ thread.start();
+ threadPool.add(worker);
+ }
+ }
+
+ public synchronized ReplicationTask selectTask() {
+ for (ReplicationTask task : queue.values()) {
+ if (task.getStatus() == Status.QUEUED) {
+ if (containerSet.getContainer(task.getContainerId()) == null) {
+ task.setStatus(Status.DOWNLOADING);
+ return task;
+ } else {
+ LOG.debug("Container {} has already been downloaded.",
+ task.getContainerId());
+ queue.remove(task.getContainerId());
+ }
+ } else if (task.getStatus() == Status.FAILED) {
+ LOG.error(
+ "Container {} can't be downloaded from any of the datanodes.",
+ task.getContainerId());
+ queue.remove(task.getContainerId());
+ } else if (task.getStatus() == Status.DONE) {
+ queue.remove(task.getContainerId());
+ LOG.info("Container {} is replicated.", task.getContainerId());
+ }
+ }
+ //no available task.
+ return null;
+ }
+
+ public void stop() {
+ for (Worker worker : threadPool) {
+ worker.stop();
+ }
+ }
+
+ @VisibleForTesting
+ public int getQueueSize() {
+ return queue.size();
+ }
+
+ private class Worker implements Runnable {
+
+ private boolean running = true;
+
+ @Override
+ public void run() {
+ try {
+ while (running) {
+ ReplicationTask task = selectTask();
+ if (task == null) {
+ synchronized (threadPool) {
+ threadPool.wait();
+ }
+ } else {
+ replicator.replicate(task);
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("Error on doing replication", ex);
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ LOG.error("Error on waiting after failed replication task", e);
+ }
+ }
+ }
+
+ public void stop() {
+ running = false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
new file mode 100644
index 0000000..9019811
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+/**
+ * The task to download a container from the sources.
+ */
+public class ReplicationTask {
+
+ private volatile Status status = Status.QUEUED;
+
+ private final long containerId;
+
+ private List<DatanodeDetails> sources;
+
+ private final Instant queued = Instant.now();
+
+ public ReplicationTask(long containerId,
+ List<DatanodeDetails> sources) {
+ this.containerId = containerId;
+ this.sources = sources;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReplicationTask that = (ReplicationTask) o;
+ return containerId == that.containerId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(containerId);
+ }
+
+ public long getContainerId() {
+ return containerId;
+ }
+
+ public List<DatanodeDetails> getSources() {
+ return sources;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(
+ Status status) {
+ this.status = status;
+ }
+
+ @Override
+ public String toString() {
+ return "ReplicationTask{" +
+ "status=" + status +
+ ", containerId=" + containerId +
+ ", sources=" + sources +
+ ", queued=" + queued +
+ '}';
+ }
+
+ public Instant getQueued() {
+ return queued;
+ }
+
+ /**
+ * Status of the replication.
+ */
+ public enum Status {
+ QUEUED,
+ DOWNLOADING,
+ FAILED,
+ DONE
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
deleted file mode 100644
index 6529922..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.replication.ContainerDownloader;
-import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.TestGenericTestUtils;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Test replication command handler.
- */
-public class TestReplicateContainerCommandHandler {
-
- private static final String EXCEPTION_MESSAGE = "Oh my god";
- private ReplicateContainerCommandHandler handler;
- private StubDownloader downloader;
- private ReplicateContainerCommand command;
- private List<Long> importedContainerIds;
-
- @Before
- public void init() {
- importedContainerIds = new ArrayList<>();
-
- OzoneConfiguration conf = new OzoneConfiguration();
- ContainerSet containerSet = Mockito.mock(ContainerSet.class);
- ContainerDispatcher containerDispatcher =
- Mockito.mock(ContainerDispatcher.class);
-
- downloader = new StubDownloader();
-
- handler = new ReplicateContainerCommandHandler(conf, containerSet,
- containerDispatcher, downloader) {
- @Override
- protected void importContainer(long containerID, Path tarFilePath) {
- importedContainerIds.add(containerID);
- }
- };
-
- //the command
- ArrayList<DatanodeDetails> datanodeDetails = new ArrayList<>();
- datanodeDetails.add(Mockito.mock(DatanodeDetails.class));
- datanodeDetails.add(Mockito.mock(DatanodeDetails.class));
-
- command = new ReplicateContainerCommand(1L, datanodeDetails);
- }
-
- @Test
- public void handle() throws TimeoutException, InterruptedException {
- //GIVEN
-
- //WHEN
- handler.handle(command, null, Mockito.mock(StateContext.class), null);
-
- TestGenericTestUtils
- .waitFor(() -> downloader.futureByContainers.size() == 1, 100, 2000);
-
- Assert.assertNotNull(downloader.futureByContainers.get(1L));
- downloader.futureByContainers.get(1L).complete(Paths.get("/tmp/test"));
-
- TestGenericTestUtils
- .waitFor(() -> importedContainerIds.size() == 1, 100, 2000);
- }
-
- @Test
- public void handleWithErrors() throws TimeoutException, InterruptedException {
- //GIVEN
- GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
- .captureLogs(ReplicateContainerCommandHandler.LOG);
-
- //WHEN
- handler.handle(command, null, Mockito.mock(StateContext.class), null);
-
- //THEN
- TestGenericTestUtils
- .waitFor(() -> downloader.futureByContainers.size() == 1, 100, 2000);
-
- Assert.assertNotNull(downloader.futureByContainers.get(1L));
- downloader.futureByContainers.get(1L)
- .completeExceptionally(new IllegalArgumentException(
- EXCEPTION_MESSAGE));
-
- TestGenericTestUtils
- .waitFor(() -> {
- String output = logCapturer.getOutput();
- return output.contains("unsuccessful") && output
- .contains(EXCEPTION_MESSAGE); },
- 100,
- 2000);
- }
-
- /**
- * Can't handle a command if there are no source replicas.
- */
- @Test(expected = IllegalArgumentException.class)
- public void handleWithoutReplicas()
- throws TimeoutException, InterruptedException {
- //GIVEN
- ReplicateContainerCommand commandWithoutReplicas =
- new ReplicateContainerCommand(1L, new ArrayList<>());
-
- //WHEN
- handler
- .handle(commandWithoutReplicas,
- null,
- Mockito.mock(StateContext.class),
- null);
-
- }
- private static class StubDownloader implements ContainerDownloader {
-
- private Map<Long, CompletableFuture<Path>> futureByContainers =
- new HashMap<>();
-
- @Override
- public void close() {
-
- }
-
- @Override
- public CompletableFuture<Path> getContainerDataFromReplicas(
- long containerId, List<DatanodeDetails> sources) {
- CompletableFuture<Path> future = new CompletableFuture<>();
- futureByContainers.put(containerId, future);
- return future;
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
new file mode 100644
index 0000000..d433319
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test the replication supervisor.
+ */
+public class TestReplicationSupervisor {
+
+ private OzoneConfiguration conf = new OzoneConfiguration();
+
+ @Test
+ public void normal() {
+ //GIVEN
+ ContainerSet set = new ContainerSet();
+
+ FakeReplicator replicator = new FakeReplicator(set);
+ ReplicationSupervisor supervisor =
+ new ReplicationSupervisor(set, replicator, 5);
+
+ List<DatanodeDetails> datanodes = IntStream.range(1, 3)
+ .mapToObj(v -> Mockito.mock(DatanodeDetails.class))
+ .collect(Collectors.toList());
+
+ try {
+ supervisor.start();
+ //WHEN
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(2L, datanodes));
+ supervisor.addTask(new ReplicationTask(2L, datanodes));
+ supervisor.addTask(new ReplicationTask(3L, datanodes));
+ try {
+ Thread.sleep(300);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ //THEN
+ System.out.println(replicator.replicated.get(0));
+
+ Assert
+ .assertEquals(3, replicator.replicated.size());
+
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @Test
+ public void duplicateMessageAfterAWhile() throws InterruptedException {
+ //GIVEN
+ ContainerSet set = new ContainerSet();
+
+ FakeReplicator replicator = new FakeReplicator(set);
+ ReplicationSupervisor supervisor =
+ new ReplicationSupervisor(set, replicator, 2);
+
+ List<DatanodeDetails> datanodes = IntStream.range(1, 3)
+ .mapToObj(v -> Mockito.mock(DatanodeDetails.class))
+ .collect(Collectors.toList());
+
+ try {
+ supervisor.start();
+ //WHEN
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ Thread.sleep(400);
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ Thread.sleep(300);
+
+ //THEN
+ System.out.println(replicator.replicated.get(0));
+
+ Assert
+ .assertEquals(1, replicator.replicated.size());
+
+ //the last item is still in the queue as we cleanup the queue during the
+ // selection
+ Assert.assertEquals(1, supervisor.getQueueSize());
+
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ private class FakeReplicator implements ContainerReplicator {
+
+ private List<ReplicationTask> replicated = new ArrayList<>();
+
+ private ContainerSet containerSet;
+
+ FakeReplicator(ContainerSet set) {
+ this.containerSet = set;
+ }
+
+ @Override
+ public void replicate(ReplicationTask task) {
+ KeyValueContainerData kvcd =
+ new KeyValueContainerData(task.getContainerId(), 100L);
+ KeyValueContainer kvc =
+ new KeyValueContainer(kvcd, conf);
+ try {
+ //download is slow
+ Thread.sleep(100);
+ replicated.add(task);
+ containerSet.addContainer(kvc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..5c905e0
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Tests for the container replication.
+ */
+package org.apache.hadoop.ozone.container.replication;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HADOOP-15726. Create utility to
limit frequency of log statements. Contributed by Erik Krogen.
Posted by xk...@apache.org.
HADOOP-15726. Create utility to limit frequency of log statements. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a30b4f9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a30b4f9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a30b4f9e
Branch: refs/heads/HDFS-12943
Commit: a30b4f9e71cf53f79c38878d6cbe5bbe79bcb277
Parents: 98c9bc4
Author: Chen Liang <cl...@apache.org>
Authored: Wed Sep 19 13:22:37 2018 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Wed Sep 19 13:22:37 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/log/LogThrottlingHelper.java | 358 +++++++++++++++++++
.../hadoop/log/TestLogThrottlingHelper.java | 172 +++++++++
.../hdfs/server/namenode/FSNamesystemLock.java | 46 +--
3 files changed, 547 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
new file mode 100644
index 0000000..aa4e61c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
@@ -0,0 +1,358 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.log;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.hadoop.util.Timer;
+
+/**
+ * This is a class to help easily throttle log statements, so that they will
+ * not be emitted more frequently than a certain rate. It is useful to help
+ * prevent flooding the application logs with redundant messages.
+ *
+ * The instantiator specifies a minimum period at which statements should be
+ * logged. When {@link #record(double...)} is called, if enough time has elapsed
+ * since the last time it was called, the return value will indicate to the
+ * caller that it should write to its actual log. Note that this class does not
+ * write to any actual log; it only records information about how many times
+ * {@code record} has been called and with what arguments, and indicates to the
+ * caller whether or not it should write to its log. If not enough time has yet
+ * elapsed, this class records the arguments and updates its summary
+ * information, and indicates to the caller that it should not log.
+ *
+ * For example, say that you want to know whenever too large of a request is
+ * received, but want to avoid flooding the logs if many such requests are
+ * received.
+ * <pre>{@code
+ * // Helper with a minimum period of 5 seconds
+ * private LogThrottlingHelper helper = new LogThrottlingHelper(5000);
+ *
+ * public void receiveRequest(int requestedObjects) {
+ * if (requestedObjects > MAXIMUM_REQUEST_SIZE) {
+ * LogAction logAction = helper.record(requestedObjects);
+ * if (logAction.shouldLog()) {
+ * LOG.warn("Received {} large request(s) with a total of {} objects " +
+ * "requested; maximum objects requested was {}",
+ * logAction.getCount(), logAction.getStats(0).getSum(),
+ * logAction.getStats(0).getMax());
+ * }
+ * }
+ * }
+ * }</pre>
+ * The above snippet allows you to record extraneous events, but if they become
+ * frequent, to limit their presence in the log to only every 5 seconds while
+ * still maintaining overall information about how many large requests were
+ * received.
+ *
+ * <p/>This class can also be used to coordinate multiple logging points; see
+ * {@link #record(String, long, double...)} for more details.
+ *
+ * <p/>This class is not thread-safe.
+ */
+public class LogThrottlingHelper {
+
+ /**
+ * An indication of what action the caller should take. If
+ * {@link #shouldLog()} is false, no other action should be taken, and it is
+ * an error to try to access any of the summary information. If
+ * {@link #shouldLog()} is true, then the caller should write to its log, and
+ * can use the {@link #getCount()} and {@link #getStats(int)} methods to
+ * determine summary information about what has been recorded into this
+ * helper.
+ *
+ * All summary information in this action only represents
+ * {@link #record(double...)} statements which were called <i>after</i> the
+ * last time the caller logged something; that is, since the last time a log
+ * action was returned with a true value for {@link #shouldLog()}. Information
+ * about the {@link #record(double...)} statement which created this log
+ * action is included.
+ */
+ public interface LogAction {
+
+ /**
+ * Return the number of records encapsulated in this action; that is, the
+ * number of times {@code record} was called to produce this action,
+ * including the current one.
+ */
+ int getCount();
+
+ /**
+ * Return summary information for the value that was recorded at index
+ * {@code idx}. Corresponds to the ordering of values passed to
+ * {@link #record(double...)}.
+ */
+ SummaryStatistics getStats(int idx);
+
+ /**
+ * If this is true, the caller should write to its log. Otherwise, the
+ * caller should take no action, and it is an error to call other methods
+ * on this object.
+ */
+ boolean shouldLog();
+
+ }
+
+ /**
+ * A {@link LogAction} representing a state that should not yet be logged.
+ * If any attempt is made to extract information from this, it will throw
+ * an {@link IllegalStateException}.
+ */
+ public static final LogAction DO_NOT_LOG = new NoLogAction();
+ private static final String DEFAULT_RECORDER_NAME =
+ "__DEFAULT_RECORDER_NAME__";
+
+ /**
+ * This throttler will not trigger log statements more frequently than this
+ * period.
+ */
+ private final long minLogPeriodMs;
+ /**
+ * The name of the recorder treated as the primary; this is the only one which
+ * will trigger logging. Other recorders are dependent on the state of this
+ * recorder. This may be null, in which case a primary has not yet been set.
+ */
+ private String primaryRecorderName;
+ private final Timer timer;
+ private final Map<String, LoggingAction> currentLogs;
+
+ private long lastLogTimestampMs = Long.MIN_VALUE;
+
+ /**
+ * Create a log helper without any primary recorder.
+ *
+ * @see #LogThrottlingHelper(long, String)
+ */
+ public LogThrottlingHelper(long minLogPeriodMs) {
+ this(minLogPeriodMs, null);
+ }
+
+ /**
+ * Create a log helper with a specified primary recorder name; this can be
+ * used in conjunction with {@link #record(String, long, double...)} to set up
+ * primary and dependent recorders. See
+ * {@link #record(String, long, double...)} for more details.
+ *
+ * @param minLogPeriodMs The minimum period with which to log; do not log
+ * more frequently than this.
+ * @param primaryRecorderName The name of the primary recorder.
+ */
+ public LogThrottlingHelper(long minLogPeriodMs, String primaryRecorderName) {
+ this(minLogPeriodMs, primaryRecorderName, new Timer());
+ }
+
+ @VisibleForTesting
+ LogThrottlingHelper(long minLogPeriodMs, String primaryRecorderName,
+ Timer timer) {
+ this.minLogPeriodMs = minLogPeriodMs;
+ this.primaryRecorderName = primaryRecorderName;
+ this.timer = timer;
+ this.currentLogs = new HashMap<>();
+ }
+
+ /**
+ * Record some set of values at the current time into this helper. Note that
+ * this does <i>not</i> actually write information to any log. Instead, this
+ * will return a LogAction indicating whether or not the caller should write
+ * to its own log. The LogAction will additionally contain summary information
+ * about the values specified since the last time the caller was expected to
+ * write to its log.
+ *
+ * <p/>Specifying multiple values will maintain separate summary statistics
+ * about each value. For example:
+ * <pre>{@code
+ * helper.record(1, 0);
+ * LogAction action = helper.record(3, 100);
+ * action.getStats(0); // == 2
+ * action.getStats(1); // == 50
+ * }</pre>
+ *
+ * @param values The values about which to maintain summary information. Every
+ * time this method is called, the same number of values must
+ * be specified.
+ * @return A LogAction indicating whether or not the caller should write to
+ * its log.
+ */
+ public LogAction record(double... values) {
+ return record(DEFAULT_RECORDER_NAME, timer.monotonicNow(), values);
+ }
+
+ /**
+ * Record some set of values at the specified time into this helper. This can
+ * be useful to avoid fetching the current time twice if the caller has
+ * already done so for other purposes. This additionally allows the caller to
+ * specify a name for this recorder. When multiple names are used, one is
+ * denoted as the primary recorder. Only recorders named as the primary
+ * will trigger logging; other names not matching the primary can <i>only</i>
+ * be triggered by following the primary. This is used to coordinate multiple
+ * logging points. A primary can be set via the
+ * {@link #LogThrottlingHelper(long, String)} constructor. If no primary
+ * is set in the constructor, then the first recorder name used becomes the
+ * primary.
+ *
+ * If multiple names are used, they maintain entirely different sets of values
+ * and summary information. For example:
+ * <pre>{@code
+ * // Initialize "pre" as the primary recorder name
+ * LogThrottlingHelper helper = new LogThrottlingHelper(1000, "pre");
+ * LogAction preLog = helper.record("pre", Time.monotonicNow());
+ * if (preLog.shouldLog()) {
+ * // ...
+ * }
+ * double eventsProcessed = ... // perform some action
+ * LogAction postLog =
+ * helper.record("post", Time.monotonicNow(), eventsProcessed);
+ * if (postLog.shouldLog()) {
+ * // ...
+ * // Can use postLog.getStats(0) to access eventsProcessed information
+ * }
+ * }</pre>
+ * Since "pre" is the primary recorder name, logging to "pre" will trigger a
+ * log action if enough time has elapsed. This will indicate that "post"
+ * should log as well. This ensures that "post" is always logged in the same
+ * iteration as "pre", yet each one is able to maintain its own summary
+ * information.
+ *
+ * <p/>Other behavior is the same as {@link #record(double...)}.
+ *
+ * @param recorderName The name of the recorder. This is used to check if the
+ * current recorder is the primary. Other names are
+ * arbitrary and are only used to differentiate between
+ * distinct recorders.
+ * @param currentTimeMs The current time.
+ * @param values The values to log.
+ * @return The LogAction for the specified recorder.
+ *
+ * @see #record(double...)
+ */
+ public LogAction record(String recorderName, long currentTimeMs,
+ double... values) {
+ if (primaryRecorderName == null) {
+ primaryRecorderName = recorderName;
+ }
+ LoggingAction currentLog = currentLogs.get(recorderName);
+ if (currentLog == null || currentLog.hasLogged()) {
+ currentLog = new LoggingAction(values.length);
+ if (!currentLogs.containsKey(recorderName)) {
+ // Always log newly created loggers
+ currentLog.setShouldLog();
+ }
+ currentLogs.put(recorderName, currentLog);
+ }
+ currentLog.recordValues(values);
+ if (primaryRecorderName.equals(recorderName) &&
+ currentTimeMs - minLogPeriodMs >= lastLogTimestampMs) {
+ lastLogTimestampMs = currentTimeMs;
+ for (LoggingAction log : currentLogs.values()) {
+ log.setShouldLog();
+ }
+ }
+ if (currentLog.shouldLog()) {
+ currentLog.setHasLogged();
+ return currentLog;
+ } else {
+ return DO_NOT_LOG;
+ }
+ }
+
+ /**
+ * A standard log action which keeps track of all of the values which have
+ * been logged. This is also used for internal bookkeeping via its private
+ * fields and methods; it will maintain whether or not it is ready to be
+ * logged ({@link #shouldLog()}) as well as whether or not it has been
+ * returned for logging yet ({@link #hasLogged()}).
+ */
+ private static class LoggingAction implements LogAction {
+
+ private int count = 0;
+ private final SummaryStatistics[] stats;
+ private boolean shouldLog = false;
+ private boolean hasLogged = false;
+
+ LoggingAction(int valueCount) {
+ stats = new SummaryStatistics[valueCount];
+ for (int i = 0; i < stats.length; i++) {
+ stats[i] = new SummaryStatistics();
+ }
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public SummaryStatistics getStats(int idx) {
+ if (idx < 0 || idx >= stats.length) {
+ throw new IllegalArgumentException("Requested stats at idx " + idx +
+ " but this log only maintains " + stats.length + " stats");
+ }
+ return stats[idx];
+ }
+
+ public boolean shouldLog() {
+ return shouldLog;
+ }
+
+ private void setShouldLog() {
+ shouldLog = true;
+ }
+
+ private boolean hasLogged() {
+ return hasLogged;
+ }
+
+ private void setHasLogged() {
+ hasLogged = true;
+ }
+
+ private void recordValues(double... values) {
+ if (values.length != stats.length) {
+ throw new IllegalArgumentException("received " + values.length +
+ " values but expected " + stats.length);
+ }
+ count++;
+ for (int i = 0; i < values.length; i++) {
+ stats[i].addValue(values[i]);
+ }
+ }
+
+ }
+
+ /**
+ * A non-logging action.
+ *
+ * @see #DO_NOT_LOG
+ */
+ private static class NoLogAction implements LogAction {
+
+ public int getCount() {
+ throw new IllegalStateException("Cannot be logged yet!");
+ }
+
+ public SummaryStatistics getStats(int idx) {
+ throw new IllegalStateException("Cannot be logged yet!");
+ }
+
+ public boolean shouldLog() {
+ return false;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
new file mode 100644
index 0000000..a675d0a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.log;
+
+import org.apache.hadoop.log.LogThrottlingHelper.LogAction;
+import org.apache.hadoop.util.FakeTimer;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link LogThrottlingHelper}.
+ */
+public class TestLogThrottlingHelper {
+
+ private static final int LOG_PERIOD = 100;
+
+ private LogThrottlingHelper helper;
+ private FakeTimer timer;
+
+ @Before
+ public void setup() {
+ timer = new FakeTimer();
+ helper = new LogThrottlingHelper(LOG_PERIOD, null, timer);
+ }
+
+ @Test
+ public void testBasicLogging() {
+ assertTrue(helper.record().shouldLog());
+
+ for (int i = 0; i < 5; i++) {
+ timer.advance(LOG_PERIOD / 10);
+ assertFalse(helper.record().shouldLog());
+ }
+ timer.advance(LOG_PERIOD);
+ assertTrue(helper.record().shouldLog());
+ }
+
+ @Test
+ public void testLoggingWithValue() {
+ assertTrue(helper.record(1).shouldLog());
+
+ for (int i = 0; i < 4; i++) {
+ timer.advance(LOG_PERIOD / 5);
+ assertFalse(helper.record(i % 2 == 0 ? 0 : 1).shouldLog());
+ }
+
+ timer.advance(LOG_PERIOD);
+ LogAction action = helper.record(0.5);
+ assertTrue(action.shouldLog());
+ assertEquals(5, action.getCount());
+ assertEquals(0.5, action.getStats(0).getMean(), 0.01);
+ assertEquals(1.0, action.getStats(0).getMax(), 0.01);
+ assertEquals(0.0, action.getStats(0).getMin(), 0.01);
+ }
+
+ @Test
+ public void testLoggingWithMultipleValues() {
+ assertTrue(helper.record(1).shouldLog());
+
+ for (int i = 0; i < 4; i++) {
+ timer.advance(LOG_PERIOD / 5);
+ int base = i % 2 == 0 ? 0 : 1;
+ assertFalse(helper.record(base, base * 2).shouldLog());
+ }
+
+ timer.advance(LOG_PERIOD);
+ LogAction action = helper.record(0.5, 1.0);
+ assertTrue(action.shouldLog());
+ assertEquals(5, action.getCount());
+ for (int i = 1; i <= 2; i++) {
+ assertEquals(0.5 * i, action.getStats(i - 1).getMean(), 0.01);
+ assertEquals(1.0 * i, action.getStats(i - 1).getMax(), 0.01);
+ assertEquals(0.0, action.getStats(i - 1).getMin(), 0.01);
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testLoggingWithInconsistentValues() {
+ assertTrue(helper.record(1, 2).shouldLog());
+ helper.record(1, 2);
+ helper.record(1, 2, 3);
+ }
+
+ @Test
+ public void testNamedLoggersWithoutSpecifiedPrimary() {
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0).shouldLog());
+
+ assertFalse(helper.record("foo", LOG_PERIOD / 2).shouldLog());
+ assertFalse(helper.record("bar", LOG_PERIOD / 2).shouldLog());
+
+ assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
+
+ assertFalse(helper.record("foo", (LOG_PERIOD * 3) / 2).shouldLog());
+ assertFalse(helper.record("bar", (LOG_PERIOD * 3) / 2).shouldLog());
+
+ assertFalse(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("foo", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ }
+
+ @Test
+ public void testPrimaryAndDependentLoggers() {
+ helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
+
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0).shouldLog());
+ assertFalse(helper.record("bar", 0).shouldLog());
+ assertFalse(helper.record("foo", 0).shouldLog());
+
+ assertFalse(helper.record("foo", LOG_PERIOD / 2).shouldLog());
+ assertFalse(helper.record("bar", LOG_PERIOD / 2).shouldLog());
+
+ // Both should log once the period has elapsed
+ assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
+
+ // "bar" should not log yet because "foo" hasn't been triggered
+ assertFalse(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("foo", LOG_PERIOD * 2).shouldLog());
+ // The timing of "bar" shouldn't matter as it is dependent on "foo"
+ assertTrue(helper.record("bar", 0).shouldLog());
+ }
+
+ @Test
+ public void testMultipleLoggersWithValues() {
+ helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
+
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0, 2).shouldLog());
+ assertTrue(helper.record("baz", 0, 3, 3).shouldLog());
+
+ // "bar"/"baz" should not log yet because "foo" hasn't been triggered
+ assertFalse(helper.record("bar", LOG_PERIOD, 2).shouldLog());
+ assertFalse(helper.record("baz", LOG_PERIOD, 3, 3).shouldLog());
+
+ // All should log once the period has elapsed
+ LogAction foo = helper.record("foo", LOG_PERIOD);
+ LogAction bar = helper.record("bar", LOG_PERIOD, 2);
+ LogAction baz = helper.record("baz", LOG_PERIOD, 3, 3);
+ assertTrue(foo.shouldLog());
+ assertTrue(bar.shouldLog());
+ assertTrue(baz.shouldLog());
+ assertEquals(1, foo.getCount());
+ assertEquals(2, bar.getCount());
+ assertEquals(2, baz.getCount());
+ assertEquals(2.0, bar.getStats(0).getMean(), 0.01);
+ assertEquals(3.0, baz.getStats(0).getMean(), 0.01);
+ assertEquals(3.0, baz.getStats(1).getMean(), 0.01);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index 5992e54..7c28465 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.log.LogThrottlingHelper;
import org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Timer;
@@ -40,6 +41,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORT
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.log.LogThrottlingHelper.LogAction;
/**
* Mimics a ReentrantReadWriteLock but does not directly implement the interface
@@ -74,11 +76,8 @@ class FSNamesystemLock {
private final long writeLockReportingThresholdMs;
/** Last time stamp for write lock. Keep the longest one for multi-entrance.*/
private long writeLockHeldTimeStampNanos;
- private int numWriteLockWarningsSuppressed = 0;
- /** Time stamp (ms) of the last time a write lock report was written. */
- private long timeStampOfLastWriteLockReportMs = 0;
- /** Longest time (ms) a write lock was held since the last report. */
- private long longestWriteLockHeldIntervalMs = 0;
+ /** Frequency limiter used for reporting long write lock hold times. */
+ private final LogThrottlingHelper writeLockReportLogger;
/** Threshold (ms) for long holding read lock report. */
private final long readLockReportingThresholdMs;
@@ -132,6 +131,8 @@ class FSNamesystemLock {
this.lockSuppressWarningIntervalMs = conf.getTimeDuration(
DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY,
DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
+ this.writeLockReportLogger =
+ new LogThrottlingHelper(lockSuppressWarningIntervalMs);
this.metricsEnabled = conf.getBoolean(
DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY,
DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT);
@@ -251,25 +252,11 @@ class FSNamesystemLock {
final long writeLockIntervalMs =
TimeUnit.NANOSECONDS.toMillis(writeLockIntervalNanos);
- boolean logReport = false;
- int numSuppressedWarnings = 0;
- long longestLockHeldIntervalMs = 0;
+ LogAction logAction = LogThrottlingHelper.DO_NOT_LOG;
if (needReport &&
writeLockIntervalMs >= this.writeLockReportingThresholdMs) {
- if (writeLockIntervalMs > longestWriteLockHeldIntervalMs) {
- longestWriteLockHeldIntervalMs = writeLockIntervalMs;
- }
- if (currentTimeMs - timeStampOfLastWriteLockReportMs >
- this.lockSuppressWarningIntervalMs) {
- logReport = true;
- numSuppressedWarnings = numWriteLockWarningsSuppressed;
- numWriteLockWarningsSuppressed = 0;
- longestLockHeldIntervalMs = longestWriteLockHeldIntervalMs;
- longestWriteLockHeldIntervalMs = 0;
- timeStampOfLastWriteLockReportMs = currentTimeMs;
- } else {
- numWriteLockWarningsSuppressed++;
- }
+ logAction = writeLockReportLogger
+ .record("write", currentTimeMs, writeLockIntervalMs);
}
coarseLock.writeLock().unlock();
@@ -278,13 +265,14 @@ class FSNamesystemLock {
addMetric(opName, writeLockIntervalNanos, true);
}
- if (logReport) {
- FSNamesystem.LOG.info("FSNamesystem write lock held for " +
- writeLockIntervalMs + " ms via\n" +
- StringUtils.getStackTrace(Thread.currentThread()) +
- "\tNumber of suppressed write-lock reports: " +
- numSuppressedWarnings + "\n\tLongest write-lock held interval: " +
- longestLockHeldIntervalMs);
+ if (logAction.shouldLog()) {
+ FSNamesystem.LOG.info("FSNamesystem write lock held for {} ms via {}\t" +
+ "Number of suppressed write-lock reports: {}\n\tLongest write-lock " +
+ "held interval: {} \n\tTotal suppressed write-lock held time: {}",
+ writeLockIntervalMs,
+ StringUtils.getStackTrace(Thread.currentThread()),
+ logAction.getCount() - 1, logAction.getStats(0).getMax(),
+ logAction.getStats(0).getSum() - writeLockIntervalMs);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDDS-488. Handle chill mode
exception from SCM in OzoneManager. Contributed by Ajay Kumar.
Posted by xk...@apache.org.
HDDS-488. Handle chill mode exception from SCM in OzoneManager. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39296537
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39296537
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39296537
Branch: refs/heads/HDFS-12943
Commit: 39296537076d19b3713a75f8453d056884c49be6
Parents: 17f5651
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Sep 18 19:58:09 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Sep 18 20:05:25 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +-
.../hadoop/ozone/om/TestScmChillMode.java | 171 +++++++++++++++++++
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 32 +++-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../hadoop/ozone/om/TestKeyManagerImpl.java | 165 ++++++++++++++++++
6 files changed, 367 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
index 0a136a7..5199ce3 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
@@ -167,7 +167,7 @@ public final class OmBucketInfo implements Auditable {
private StorageType storageType;
private long creationTime;
- Builder() {
+ public Builder() {
//Default values
this.acls = new LinkedList<>();
this.isVersionEnabled = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
index 27e25f9..165d9ab 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
@@ -151,7 +151,7 @@ public final class OmVolumeArgs implements Auditable{
/**
* Constructs a builder.
*/
- Builder() {
+ public Builder() {
keyValueMap = new HashMap<>();
aclMap = new OmOzoneAclMap();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
new file mode 100644
index 0000000..954fa0f
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.TestStorageContainerManagerHelper;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Test Ozone Manager operation in distributed handler scenario.
+ */
+public class TestScmChillMode {
+
+ private static MiniOzoneCluster cluster = null;
+ private static MiniOzoneCluster.Builder builder = null;
+ private static OzoneConfiguration conf;
+ private static OzoneManager om;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(1000 * 200);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true and
+ * OZONE_HANDLER_TYPE_KEY = "distributed"
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ builder = MiniOzoneCluster.newBuilder(conf)
+ .setHbInterval(1000)
+ .setHbProcessorInterval(500)
+ .setStartDataNodes(false);
+ cluster = builder.build();
+ cluster.startHddsDatanodes();
+ cluster.waitForClusterToBeReady();
+ om = cluster.getOzoneManager();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testChillModeOperations() throws Exception {
+ final AtomicReference<MiniOzoneCluster> miniCluster =
+ new AtomicReference<>();
+
+ try {
+ // Create {numKeys} random names keys.
+ TestStorageContainerManagerHelper helper =
+ new TestStorageContainerManagerHelper(cluster, conf);
+ Map<String, OmKeyInfo> keyLocations = helper.createKeys(100, 4096);
+ final List<ContainerInfo> containers = cluster
+ .getStorageContainerManager()
+ .getScmContainerManager().getStateManager().getAllContainers();
+ GenericTestUtils.waitFor(() -> {
+ return containers.size() > 10;
+ }, 100, 1000);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ String keyName = "key" + RandomStringUtils.randomNumeric(5);
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(1000)
+ .build();
+ OmVolumeArgs volArgs = new OmVolumeArgs.Builder()
+ .setAdminName(adminName)
+ .setCreationTime(Time.monotonicNow())
+ .setQuotaInBytes(10000)
+ .setVolume(volumeName)
+ .setOwnerName(userName)
+ .build();
+ OmBucketInfo bucketInfo = new OmBucketInfo.Builder()
+ .setBucketName(bucketName)
+ .setIsVersionEnabled(false)
+ .setVolumeName(volumeName)
+ .build();
+ om.createVolume(volArgs);
+ om.createBucket(bucketInfo);
+ om.openKey(keyArgs);
+ //om.commitKey(keyArgs, 1);
+
+ cluster.stop();
+
+ new Thread(() -> {
+ try {
+ miniCluster.set(builder.build());
+ } catch (IOException e) {
+ fail("failed");
+ }
+ }).start();
+
+ StorageContainerManager scm;
+ GenericTestUtils.waitFor(() -> {
+ return miniCluster.get() != null;
+ }, 100, 1000 * 3);
+
+ scm = miniCluster.get().getStorageContainerManager();
+ Assert.assertTrue(scm.isInChillMode());
+
+ om = miniCluster.get().getOzoneManager();
+
+ LambdaTestUtils.intercept(OMException.class,
+ "ChillModePrecheck failed for allocateBlock",
+ () -> om.openKey(keyArgs));
+
+ } finally {
+ if (miniCluster.get() != null) {
+ try {
+ miniCluster.get().shutdown();
+ } catch (Exception e) {
+ // do nothing.
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index c14d0d8..41b391a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -155,9 +156,18 @@ public class KeyManagerImpl implements KeyManager {
}
OmKeyInfo keyInfo =
OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
- AllocatedBlock allocatedBlock =
- scmBlockClient.allocateBlock(scmBlockSize, keyInfo.getType(),
- keyInfo.getFactor(), omId);
+ AllocatedBlock allocatedBlock;
+ try {
+ allocatedBlock =
+ scmBlockClient.allocateBlock(scmBlockSize, keyInfo.getType(),
+ keyInfo.getFactor(), omId);
+ } catch (SCMException ex) {
+ if (ex.getResult()
+ .equals(SCMException.ResultCodes.CHILL_MODE_EXCEPTION)) {
+ throw new OMException(ex.getMessage(), ResultCodes.SCM_IN_CHILL_MODE);
+ }
+ throw ex;
+ }
OmKeyLocationInfo info = new OmKeyLocationInfo.Builder()
.setBlockID(allocatedBlock.getBlockID())
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
@@ -208,8 +218,20 @@ public class KeyManagerImpl implements KeyManager {
// some blocks and piggyback to client, to save RPC calls.
while (requestedSize > 0) {
long allocateSize = Math.min(scmBlockSize, requestedSize);
- AllocatedBlock allocatedBlock =
- scmBlockClient.allocateBlock(allocateSize, type, factor, omId);
+ AllocatedBlock allocatedBlock;
+ try {
+ allocatedBlock = scmBlockClient
+ .allocateBlock(allocateSize, type, factor, omId);
+ } catch (IOException ex) {
+ if (ex instanceof SCMException) {
+ if (((SCMException) ex).getResult()
+ .equals(SCMException.ResultCodes.CHILL_MODE_EXCEPTION)) {
+ throw new OMException(ex.getMessage(),
+ ResultCodes.SCM_IN_CHILL_MODE);
+ }
+ }
+ throw ex;
+ }
OmKeyLocationInfo subKeyInfo = new OmKeyLocationInfo.Builder()
.setBlockID(allocatedBlock.getBlockID())
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 55cef97..393ac91 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@ -113,6 +113,7 @@ public class OMException extends IOException {
FAILED_METADATA_ERROR,
FAILED_INTERNAL_ERROR,
OM_NOT_INITIALIZED,
- SCM_VERSION_MISMATCH_ERROR
+ SCM_VERSION_MISMATCH_ERROR,
+ SCM_IN_CHILL_MODE
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
new file mode 100644
index 0000000..d722155
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
+import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.db.RDBStore;
+import org.apache.hadoop.utils.db.Table;
+import org.apache.hadoop.utils.db.TableConfig;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.Statistics;
+import org.rocksdb.StatsLevel;
+
+/**
+ * Test class for @{@link KeyManagerImpl}.
+ * */
+public class TestKeyManagerImpl {
+
+ private static KeyManagerImpl keyManager;
+ private static ScmBlockLocationProtocol scmBlockLocationProtocol;
+ private static OzoneConfiguration conf;
+ private static OMMetadataManager metadataManager;
+ private static long blockSize = 1000;
+ private static final String KEY_NAME = "key1";
+ private static final String BUCKET_NAME = "bucket1";
+ private static final String VOLUME_NAME = "vol1";
+ private static RDBStore rdbStore = null;
+ private static Table rdbTable = null;
+ private static DBOptions options = null;
+ private KeyInfo keyData;
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new OzoneConfiguration();
+ scmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class);
+ metadataManager = Mockito.mock(OMMetadataManager.class);
+ keyManager = new KeyManagerImpl(scmBlockLocationProtocol, metadataManager,
+ conf, "om1");
+ setupMocks();
+ }
+
+ private void setupMocks() throws Exception {
+ Mockito.when(scmBlockLocationProtocol
+ .allocateBlock(Mockito.anyLong(), Mockito.any(ReplicationType.class),
+ Mockito.any(ReplicationFactor.class), Mockito.anyString()))
+ .thenThrow(
+ new SCMException("ChillModePrecheck failed for allocateBlock",
+ ResultCodes.CHILL_MODE_EXCEPTION));
+ setupRocksDb();
+ Mockito.when(metadataManager.getVolumeTable()).thenReturn(rdbTable);
+ Mockito.when(metadataManager.getBucketTable()).thenReturn(rdbTable);
+ Mockito.when(metadataManager.getOpenKeyTable()).thenReturn(rdbTable);
+ Mockito.when(metadataManager.getLock())
+ .thenReturn(new OzoneManagerLock(conf));
+ Mockito.when(metadataManager.getVolumeKey(VOLUME_NAME))
+ .thenReturn(VOLUME_NAME.getBytes());
+ Mockito.when(metadataManager.getBucketKey(VOLUME_NAME, BUCKET_NAME))
+ .thenReturn(BUCKET_NAME.getBytes());
+ Mockito.when(metadataManager.getOpenKeyBytes(VOLUME_NAME, BUCKET_NAME,
+ KEY_NAME, 1)).thenReturn(KEY_NAME.getBytes());
+ }
+
+ private void setupRocksDb() throws Exception {
+ options = new DBOptions();
+ options.setCreateIfMissing(true);
+ options.setCreateMissingColumnFamilies(true);
+
+ Statistics statistics = new Statistics();
+ statistics.setStatsLevel(StatsLevel.ALL);
+ options = options.setStatistics(statistics);
+
+ Set<TableConfig> configSet = new HashSet<>();
+ for (String name : Arrays
+ .asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
+ "testTable")) {
+ TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
+ configSet.add(newConfig);
+ }
+ keyData = KeyInfo.newBuilder()
+ .setKeyName(KEY_NAME)
+ .setBucketName(BUCKET_NAME)
+ .setVolumeName(VOLUME_NAME)
+ .setDataSize(blockSize)
+ .setType(ReplicationType.STAND_ALONE)
+ .setFactor(ReplicationFactor.ONE)
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .build();
+
+ rdbStore = new RDBStore(folder.newFolder(), options, configSet);
+ rdbTable = rdbStore.getTable("testTable");
+ rdbTable.put(VOLUME_NAME.getBytes(),
+ RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8));
+ rdbTable.put(BUCKET_NAME.getBytes(),
+ RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8));
+ rdbTable.put(KEY_NAME.getBytes(), keyData.toByteArray());
+ }
+
+ @Test
+ public void allocateBlockFailureInChillMode() throws Exception {
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setKeyName(KEY_NAME)
+ .setBucketName(BUCKET_NAME)
+ .setFactor(ReplicationFactor.ONE)
+ .setType(ReplicationType.STAND_ALONE)
+ .setVolumeName(VOLUME_NAME).build();
+ LambdaTestUtils.intercept(OMException.class,
+ "ChillModePrecheck failed for allocateBlock", () -> {
+ keyManager.allocateBlock(keyArgs, 1);
+ });
+ }
+
+ @Test
+ public void openKeyFailureInChillMode() throws Exception {
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setKeyName(KEY_NAME)
+ .setBucketName(BUCKET_NAME)
+ .setFactor(ReplicationFactor.ONE)
+ .setDataSize(1000)
+ .setType(ReplicationType.STAND_ALONE)
+ .setVolumeName(VOLUME_NAME).build();
+ LambdaTestUtils.intercept(OMException.class,
+ "ChillModePrecheck failed for allocateBlock", () -> {
+ keyManager.openKey(keyArgs);
+ });
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-8771. CapacityScheduler fails
to unreserve when cluster resource contains empty resource type. Contributed
by Tao Yang.
Posted by xk...@apache.org.
YARN-8771. CapacityScheduler fails to unreserve when cluster resource contains empty resource type. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0712537e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0712537e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0712537e
Branch: refs/heads/HDFS-12943
Commit: 0712537e799bc03855d548d1f4bd690dd478b871
Parents: 28ceb34
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Sep 19 19:31:07 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Sep 19 19:31:07 2018 +0800
----------------------------------------------------------------------
.../allocator/RegularContainerAllocator.java | 3 +-
.../capacity/TestContainerAllocation.java | 68 ++++++++++++++++++++
2 files changed, 69 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712537e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 3e337ef..c0a11a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -543,8 +543,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
currentResoureLimits.getAmountNeededUnreserve());
boolean needToUnreserve =
- Resources.greaterThan(rc, clusterResource,
- resourceNeedToUnReserve, Resources.none());
+ rc.isAnyMajorResourceAboveZero(resourceNeedToUnReserve);
RMContainer unreservedContainer = null;
boolean reservationsContinueLooking =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712537e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index b9bfc2a..3d028ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -59,7 +60,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemoved
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -984,4 +988,68 @@ public class TestContainerAllocation {
Assert.assertEquals(2, lq.getMetrics().getAppsPending());
rm1.close();
}
+
+ @Test(timeout = 60000)
+ public void testUnreserveWhenClusterResourceHasEmptyResourceType()
+ throws Exception {
+ /**
+ * Test case:
+ * Create a cluster with two nodes whose node resource both are
+ * <8GB, 8core, 0>, create queue "a" whose max-resource is <8GB, 8 core, 0>,
+ * submit app1 to queue "a" whose am use <1GB, 1 core, 0> and launch on nm1,
+ * submit app2 to queue "b" whose am use <1GB, 1 core, 0> and launch on nm1,
+ * app1 asks two <7GB, 1core> containers and nm1 do 1 heartbeat,
+ * then scheduler reserves one container on nm1.
+ *
+ * After nm2 do next node heartbeat, scheduler should unreserve the reserved
+ * container on nm1 then allocate a container on nm2.
+ */
+ TestResourceUtils.addNewTypesToResources("resource1");
+ CapacitySchedulerConfiguration newConf =
+ (CapacitySchedulerConfiguration) TestUtils
+ .getConfigurationWithMultipleQueues(conf);
+ newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+ newConf
+ .setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
+ // Set maximum capacity of queue "a" to 50
+ newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 50);
+ MockRM rm1 = new MockRM(newConf);
+
+ RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
+ nodeLabelsManager.init(newConf);
+ rm1.getRMContext().setNodeLabelManager(nodeLabelsManager);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // launch an app to queue "a", AM container should be launched on nm1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // launch another app to queue "b", AM container should be launched on nm1
+ RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b");
+ MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ am1.allocate("*", 7 * GB, 2, new ArrayList<ContainerId>());
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId());
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+
+ // Do nm1 heartbeats 1 times, will reserve a container on nm1 for app1
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Assert.assertEquals(1, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(1, schedulerApp1.getReservedContainers().size());
+
+ // Do nm2 heartbeats 1 times, will unreserve a container on nm1
+ // and allocate a container on nm2 for app1
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+ Assert.assertEquals(2, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(0, schedulerApp1.getReservedContainers().size());
+
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDDS-366. Update functions
impacted by SCM chill mode in StorageContainerLocationProtocol. Contributed
by Ajay Kumar.
Posted by xk...@apache.org.
HDDS-366. Update functions impacted by SCM chill mode in StorageContainerLocationProtocol. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/295cce39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/295cce39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/295cce39
Branch: refs/heads/HDFS-12943
Commit: 295cce39edfce1e81d3a1fdcb4e70442c4f60c18
Parents: 2cf8927
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Sep 18 13:28:04 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Sep 18 13:32:48 2018 -0700
----------------------------------------------------------------------
hadoop-hdds/common/src/main/proto/hdds.proto | 7 +++
.../org/apache/hadoop/hdds/scm/ScmUtils.java | 45 +++++++++++++++
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 17 +-----
.../hdds/scm/container/ContainerMapping.java | 6 --
.../hdds/scm/server/ChillModePrecheck.java | 12 ++--
.../hdds/scm/server/SCMChillModeManager.java | 1 +
.../scm/server/SCMClientProtocolServer.java | 53 ++++++++++++++++-
.../scm/server/StorageContainerManager.java | 56 ++++++++++--------
.../scm/container/TestContainerMapping.java | 8 ---
.../scm/server/TestSCMClientProtocolServer.java | 60 ++++++++++++++++++++
.../ozone/TestStorageContainerManager.java | 55 ++++++++++++++++++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 15 +++--
12 files changed, 273 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 41f1851..dedc57b 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -176,6 +176,13 @@ enum ScmOps {
keyBlocksInfoList = 2;
getScmInfo = 3;
deleteBlock = 4;
+ createReplicationPipeline = 5;
+ allocateContainer = 6;
+ getContainer = 7;
+ getContainerWithPipeline = 8;
+ listContainer = 9;
+ deleteContainer = 10;
+ queryNode = 11;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
new file mode 100644
index 0000000..435f0a5
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.server.Precheck;
+
+/**
+ * SCM utility class.
+ */
+public final class ScmUtils {
+
+ private ScmUtils() {
+ }
+
+ /**
+ * Perform all prechecks for given scm operation.
+ *
+ * @param operation
+ * @param preChecks prechecks to be performed
+ */
+ public static void preCheck(ScmOps operation, Precheck... preChecks)
+ throws SCMException {
+ for (Precheck preCheck : preChecks) {
+ preCheck.check(operation);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 3405b0d..d383c68 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmUtils;
import org.apache.hadoop.hdds.scm.container.Mapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -30,7 +31,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.server.ChillModePrecheck;
-import org.apache.hadoop.hdds.scm.server.Precheck;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.metrics2.util.MBeans;
@@ -190,7 +190,7 @@ public class BlockManagerImpl implements EventHandler<Boolean>,
ReplicationType type, ReplicationFactor factor, String owner)
throws IOException {
LOG.trace("Size;{} , type : {}, factor : {} ", size, type, factor);
- preCheck(ScmOps.allocateBlock, chillModePrecheck);
+ ScmUtils.preCheck(ScmOps.allocateBlock, chillModePrecheck);
if (size < 0 || size > containerSize) {
LOG.warn("Invalid block size requested : {}", size);
throw new SCMException("Unsupported block size: " + size,
@@ -417,19 +417,6 @@ public class BlockManagerImpl implements EventHandler<Boolean>,
return this.blockDeletingService;
}
- /**
- * Perform all prechecks for given operations.
- *
- * @param operation
- * @param preChecks prechecks to be performed
- */
- public void preCheck(ScmOps operation, Precheck... preChecks)
- throws SCMException {
- for (Precheck preCheck : preChecks) {
- preCheck.check(operation);
- }
- }
-
@Override
public void onMessage(Boolean inChillMode, EventPublisher publisher) {
this.chillModePrecheck.setInChillMode(inChillMode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 11cc9ee..206e24b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -274,12 +274,6 @@ public class ContainerMapping implements Mapping {
ContainerInfo containerInfo;
ContainerWithPipeline containerWithPipeline;
- if (!nodeManager.isOutOfChillMode()) {
- throw new SCMException(
- "Unable to create container while in chill mode",
- SCMException.ResultCodes.CHILL_MODE_EXCEPTION);
- }
-
lock.lock();
try {
containerWithPipeline = containerStateManager.allocateContainer(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
index 81556fa..b92413e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.server;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
@@ -28,15 +29,16 @@ import org.apache.hadoop.hdds.scm.server.SCMChillModeManager.ChillModeRestricted
* */
public class ChillModePrecheck implements Precheck<ScmOps> {
- private boolean inChillMode;
+ private AtomicBoolean inChillMode = new AtomicBoolean(true);
public static final String PRECHECK_TYPE = "ChillModePrecheck";
public boolean check(ScmOps op) throws SCMException {
- if(inChillMode && ChillModeRestrictedOps.isRestrictedInChillMode(op)) {
+ if (inChillMode.get() && ChillModeRestrictedOps
+ .isRestrictedInChillMode(op)) {
throw new SCMException("ChillModePrecheck failed for " + op,
ResultCodes.CHILL_MODE_EXCEPTION);
}
- return inChillMode;
+ return inChillMode.get();
}
@Override
@@ -45,10 +47,10 @@ public class ChillModePrecheck implements Precheck<ScmOps> {
}
public boolean isInChillMode() {
- return inChillMode;
+ return inChillMode.get();
}
public void setInChillMode(boolean inChillMode) {
- this.inChillMode = inChillMode;
+ this.inChillMode.set(inChillMode);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
index 0b7bfda..b35ac1b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
@@ -234,6 +234,7 @@ public class SCMChillModeManager implements
static {
restrictedOps.add(ScmOps.allocateBlock);
+ restrictedOps.add(ScmOps.allocateContainer);
}
public static boolean isRestrictedInChillMode(ScmOps opName) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 77e495d..3f1943c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -27,15 +27,21 @@ import com.google.protobuf.BlockingService;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos;
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.ScmUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
@@ -71,13 +77,14 @@ import static org.apache.hadoop.hdds.scm.server.StorageContainerManager
* The RPC server that listens to requests from clients.
*/
public class SCMClientProtocolServer implements
- StorageContainerLocationProtocol {
+ StorageContainerLocationProtocol, EventHandler<Boolean> {
private static final Logger LOG =
LoggerFactory.getLogger(SCMClientProtocolServer.class);
private final RPC.Server clientRpcServer;
private final InetSocketAddress clientRpcAddress;
private final StorageContainerManager scm;
private final OzoneConfiguration conf;
+ private ChillModePrecheck chillModePrecheck = new ChillModePrecheck();
public SCMClientProtocolServer(OzoneConfiguration conf,
StorageContainerManager scm) throws IOException {
@@ -149,6 +156,7 @@ public class SCMClientProtocolServer implements
public ContainerWithPipeline allocateContainer(HddsProtos.ReplicationType
replicationType, HddsProtos.ReplicationFactor factor,
String owner) throws IOException {
+ ScmUtils.preCheck(ScmOps.allocateContainer, chillModePrecheck);
String remoteUser = getRpcRemoteUsername();
getScm().checkAdminAccess(remoteUser);
@@ -167,12 +175,39 @@ public class SCMClientProtocolServer implements
@Override
public ContainerWithPipeline getContainerWithPipeline(long containerID)
throws IOException {
+ if (chillModePrecheck.isInChillMode()) {
+ ContainerInfo contInfo = scm.getScmContainerManager()
+ .getContainer(containerID);
+ if (contInfo.isContainerOpen()) {
+ if (!hasRequiredReplicas(contInfo)) {
+ throw new SCMException("Open container " + containerID + " doesn't"
+ + " have enough replicas to service this operation in "
+ + "Chill mode.", ResultCodes.CHILL_MODE_EXCEPTION);
+ }
+ }
+ }
String remoteUser = getRpcRemoteUsername();
getScm().checkAdminAccess(remoteUser);
return scm.getScmContainerManager()
.getContainerWithPipeline(containerID);
}
+ /**
+ * Check if container reported replicas are equal or greater than required
+ * replication factor.
+ */
+ private boolean hasRequiredReplicas(ContainerInfo contInfo) {
+ try{
+ return getScm().getScmContainerManager().getStateManager()
+ .getContainerReplicas(contInfo.containerID())
+ .size() >= contInfo.getReplicationFactor().getNumber();
+ } catch (SCMException ex) {
+ // getContainerReplicas throws exception if no replica's exist for given
+ // container.
+ return false;
+ }
+ }
+
@Override
public List<ContainerInfo> listContainer(long startContainerID,
int count) throws IOException {
@@ -291,6 +326,22 @@ public class SCMClientProtocolServer implements
}
/**
+ * Set chill mode status based on SCMEvents.CHILL_MODE_STATUS event.
+ */
+ @Override
+ public void onMessage(Boolean inChillMOde, EventPublisher publisher) {
+ chillModePrecheck.setInChillMode(inChillMOde);
+ }
+
+ /**
+ * Set chill mode status based on .
+ */
+ public boolean getChillModeStatus() {
+ return chillModePrecheck.isInChillMode();
+ }
+
+
+ /**
* Query the System for Nodes.
*
* @param nodeState - NodeState that we are interested in matching.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 9216e5d..8e76606 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineCloseHandler;
import org.apache.hadoop.hdds.scm.pipelines.PipelineActionEventHandler;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
@@ -241,29 +242,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
PipelineCloseHandler pipelineCloseHandler =
new PipelineCloseHandler(scmContainerManager);
- eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
- eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
- eventQueue.addHandler(SCMEvents.CONTAINER_REPORT, containerReportHandler);
- eventQueue.addHandler(SCMEvents.CONTAINER_ACTIONS, actionsHandler);
- eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
- eventQueue.addHandler(SCMEvents.NEW_NODE, newNodeHandler);
- eventQueue.addHandler(SCMEvents.STALE_NODE, staleNodeHandler);
- eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
- eventQueue.addHandler(SCMEvents.CMD_STATUS_REPORT, cmdStatusReportHandler);
- eventQueue.addHandler(SCMEvents.START_REPLICATION,
- replicationStatus.getReplicationStatusListener());
- eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
- replicationStatus.getChillModeStatusListener());
- eventQueue
- .addHandler(SCMEvents.PENDING_DELETE_STATUS, pendingDeleteHandler);
- eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS,
- pipelineActionEventHandler);
- eventQueue.addHandler(SCMEvents.PIPELINE_CLOSE, pipelineCloseHandler);
- eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
- eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
- (BlockManagerImpl) scmBlockManager);
-
long watcherTimeout =
conf.getTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT,
HDDS_SCM_WATCHER_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
@@ -298,6 +276,31 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
blockProtocolServer = new SCMBlockProtocolServer(conf, this);
clientProtocolServer = new SCMClientProtocolServer(conf, this);
httpServer = new StorageContainerManagerHttpServer(conf);
+
+ eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
+ eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
+ eventQueue.addHandler(SCMEvents.CONTAINER_REPORT, containerReportHandler);
+ eventQueue.addHandler(SCMEvents.CONTAINER_ACTIONS, actionsHandler);
+ eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
+ eventQueue.addHandler(SCMEvents.NEW_NODE, newNodeHandler);
+ eventQueue.addHandler(SCMEvents.STALE_NODE, staleNodeHandler);
+ eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
+ eventQueue.addHandler(SCMEvents.CMD_STATUS_REPORT, cmdStatusReportHandler);
+ eventQueue.addHandler(SCMEvents.START_REPLICATION,
+ replicationStatus.getReplicationStatusListener());
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
+ replicationStatus.getChillModeStatusListener());
+ eventQueue
+ .addHandler(SCMEvents.PENDING_DELETE_STATUS, pendingDeleteHandler);
+ eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS,
+ pipelineActionEventHandler);
+ eventQueue.addHandler(SCMEvents.PIPELINE_CLOSE, pipelineCloseHandler);
+ eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
+ (BlockManagerImpl) scmBlockManager);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, clientProtocolServer);
+
registerMXBean();
}
@@ -830,6 +833,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return scmChillModeManager.getInChillMode();
}
+ /**
+ * Returns EventPublisher.
+ */
+ public EventPublisher getEventQueue(){
+ return eventQueue;
+ }
+
@VisibleForTesting
public double getCurrentContainerThreshold() {
return scmChillModeManager.getCurrentContainerThreshold();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
index 481f94c..224f6ddd 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
@@ -206,14 +206,6 @@ public class TestContainerMapping {
}
@Test
- public void testChillModeAllocateContainerFails() throws IOException {
- nodeManager.setChillmode(true);
- thrown.expectMessage("Unable to create container while in chill mode");
- mapping.allocateContainer(xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), containerOwner);
- }
-
- @Test
public void testContainerCreationLeaseTimeout() throws IOException,
InterruptedException {
nodeManager.setChillmode(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
new file mode 100644
index 0000000..4b20018
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.server;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for @{@link SCMClientProtocolServer}.
+ * */
+public class TestSCMClientProtocolServer {
+ private SCMClientProtocolServer scmClientProtocolServer;
+ private OzoneConfiguration config;
+ private EventQueue eventQueue;
+
+ @Before
+ public void setUp() throws Exception {
+ config = new OzoneConfiguration();
+ eventQueue = new EventQueue();
+ scmClientProtocolServer = new SCMClientProtocolServer(config, null);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, scmClientProtocolServer);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testAllocateContainerFailureInChillMode() throws Exception {
+ LambdaTestUtils.intercept(SCMException.class,
+ "hillModePrecheck failed for allocateContainer", () -> {
+ scmClientProtocolServer.allocateContainer(
+ ReplicationType.STAND_ALONE, ReplicationFactor.ONE, "");
+ });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 1364d77..3d9a043 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -43,6 +43,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -53,6 +55,7 @@ import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
import org.apache.hadoop.hdds.scm.container.ContainerMapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.server.SCMChillModeManager;
@@ -66,6 +69,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -551,4 +555,55 @@ public class TestStorageContainerManager {
cluster.shutdown();
}
+ @Test
+ public void testSCMChillModeRestrictedOp() throws Exception {
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
+ OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB);
+
+ MiniOzoneClusterImpl cluster = (MiniOzoneClusterImpl) MiniOzoneCluster
+ .newBuilder(conf)
+ .setHbInterval(1000)
+ .setHbProcessorInterval(500)
+ .setStartDataNodes(false)
+ .build();
+
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ assertTrue(scm.isInChillMode());
+
+ LambdaTestUtils.intercept(SCMException.class,
+ "ChillModePrecheck failed for allocateContainer", () -> {
+ scm.getClientProtocolServer()
+ .allocateContainer(ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE, "");
+ });
+
+ cluster.startHddsDatanodes();
+ cluster.waitForClusterToBeReady();
+ assertFalse(scm.isInChillMode());
+
+ TestStorageContainerManagerHelper helper =
+ new TestStorageContainerManagerHelper(cluster, conf);
+ helper.createKeys(10, 4096);
+ SCMClientProtocolServer clientProtocolServer = cluster
+ .getStorageContainerManager().getClientProtocolServer();
+
+ final List<ContainerInfo> containers = scm.getScmContainerManager()
+ .getStateManager().getAllContainers();
+ scm.getEventQueue().fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
+ assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
+ GenericTestUtils.waitFor(() -> {
+ return clientProtocolServer.getChillModeStatus();
+ }, 50, 1000 * 5);
+ assertTrue(clientProtocolServer.getChillModeStatus());
+
+ LambdaTestUtils.intercept(SCMException.class,
+ "Open container " + containers.get(0).getContainerID() + " "
+ + "doesn't have enough replicas to service this operation in Chill"
+ + " mode.", () -> clientProtocolServer
+ .getContainerWithPipeline(containers.get(0).getContainerID()));
+ cluster.shutdown();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 65bd036..4026348 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.scm;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -60,6 +61,8 @@ import static org.junit.Assert.assertEquals;
@RunWith(Parameterized.class)
public class TestContainerSQLCli {
+ private EventQueue eventQueue;
+
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
@@ -114,12 +117,16 @@ public class TestContainerSQLCli {
.getDatanodeDetails().getIpAddress();
cluster.getOzoneManager().stop();
cluster.getStorageContainerManager().stop();
-
+ eventQueue = new EventQueue();
nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
mapping = new ContainerMapping(conf, nodeManager, 128,
- new EventQueue());
- blockManager = new BlockManagerImpl(conf, nodeManager, mapping, null);
-
+ eventQueue);
+ blockManager = new BlockManagerImpl(conf, nodeManager, mapping, eventQueue);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager);
+ eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false);
+ GenericTestUtils.waitFor(() -> {
+ return !blockManager.isScmInChillMode();
+ }, 10, 1000 * 15);
// blockManager.allocateBlock() will create containers if there is none
// stored in levelDB. The number of containers to create is the value of
// OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE which we set to 2.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HADOOP-15759. AliyunOSS: Update
oss-sdk version to 3.0.0. Contributed by Jinhu Wu.
Posted by xk...@apache.org.
HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4fca6aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4fca6aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4fca6aa
Branch: refs/heads/HDFS-12943
Commit: e4fca6aae46a3c04fc56897986a4ab4e5aa98503
Parents: f796cfd
Author: Sammi Chen <sa...@intel.com>
Authored: Tue Sep 18 18:37:49 2018 +0800
Committer: Sammi Chen <sa...@intel.com>
Committed: Tue Sep 18 18:37:49 2018 +0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4fca6aa/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3669ffb..275ae6e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1347,7 +1347,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
- <version>2.8.3</version>
+ <version>3.0.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDFS-13908.
TestDataNodeMultipleRegistrations is flaky. Contributed by Ayush Saxena.
Posted by xk...@apache.org.
HDFS-13908. TestDataNodeMultipleRegistrations is flaky. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56e0d635
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56e0d635
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56e0d635
Branch: refs/heads/HDFS-12943
Commit: 56e0d635e0f49772d001c1fdb385110c705a622a
Parents: 61a4b07
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Sep 19 09:30:25 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Sep 19 09:30:25 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/TestDataNodeMultipleRegistrations.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e0d635/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
index 4a49477..bd28fde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
@@ -250,7 +250,7 @@ public class TestDataNodeMultipleRegistrations {
try {
cluster.startDataNodes(conf, 1, true, null, null);
// let the initialization be complete
- Thread.sleep(10000);
+ cluster.waitActive();
DataNode dn = cluster.getDataNodes().get(0);
assertTrue("Datanode should be running", dn.isDatanodeUp());
assertEquals("Only one BPOfferService should be running", 1,
@@ -274,7 +274,7 @@ public class TestDataNodeMultipleRegistrations {
try {
cluster.startDataNodes(conf, 1, true, null, null);
// let the initialization be complete
- Thread.sleep(10000);
+ cluster.waitActive();
DataNode dn = cluster.getDataNodes().get(0);
assertTrue("Datanode should be running", dn.isDatanodeUp());
assertEquals("BPOfferService should be running", 1,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-509.
TestStorageContainerManager is flaky. Contributed by Xiaoyu Yao and Ajay
Kumar.
Posted by xk...@apache.org.
HDDS-509. TestStorageContainerManager is flaky.
Contributed by Xiaoyu Yao and Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98c9bc4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98c9bc4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98c9bc4a
Branch: refs/heads/HDFS-12943
Commit: 98c9bc4aa0ef6358e1ea6ca2f18cec7ae24158dd
Parents: 090272d
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 19 12:57:44 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 19 13:03:19 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/ozone/TestStorageContainerManager.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98c9bc4a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 3d9a043..94ab6c8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -588,11 +588,10 @@ public class TestStorageContainerManager {
helper.createKeys(10, 4096);
SCMClientProtocolServer clientProtocolServer = cluster
.getStorageContainerManager().getClientProtocolServer();
-
+ assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
final List<ContainerInfo> containers = scm.getScmContainerManager()
.getStateManager().getAllContainers();
scm.getEventQueue().fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
- assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
GenericTestUtils.waitFor(() -> {
return clientProtocolServer.getChillModeStatus();
}, 50, 1000 * 5);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HADOOP-15304. [JDK10] Migrate from
com.sun.tools.doclets to the replacement. Contributed by Akira Ajisaka.
Posted by xk...@apache.org.
HADOOP-15304. [JDK10] Migrate from com.sun.tools.doclets to the replacement. Contributed by Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f796cfde
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f796cfde
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f796cfde
Branch: refs/heads/HDFS-12943
Commit: f796cfde76519c79ec5d0775e5930ed5e19b0ba6
Parents: 78a0d17
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Tue Sep 18 18:48:25 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Tue Sep 18 18:48:25 2018 +0900
----------------------------------------------------------------------
.../hadoop-annotations/pom.xml | 24 ++++++++++++++
hadoop-project-dist/pom.xml | 34 ++++++++++++++------
2 files changed, 49 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f796cfde/hadoop-common-project/hadoop-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index 64dde71..fd329e2 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -53,6 +53,30 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>jdk10</id>
+ <activation>
+ <jdk>[10,)</jdk>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>org/apache/hadoop/classification/tools/</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <excludePackageNames>org.apache.hadoop.classification.tools</excludePackageNames>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f796cfde/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 66afc14..fd67a71 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -115,15 +115,6 @@
<packages>org.apache.hadoop*</packages>
</group>
</groups>
- <doclet>org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet</doclet>
- <docletArtifacts>
- <docletArtifact>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>${hadoop.version}</version>
- </docletArtifact>
- </docletArtifacts>
- <useStandardDocletOptions>true</useStandardDocletOptions>
<!-- switch on dependency-driven aggregation -->
<includeDependencySources>false</includeDependencySources>
@@ -417,5 +408,30 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>doclet</id>
+ <activation>
+ <jdk>(,10)</jdk>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <doclet>org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet</doclet>
+ <docletArtifacts>
+ <docletArtifact>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <version>${hadoop.version}</version>
+ </docletArtifact>
+ </docletArtifacts>
+ <useStandardDocletOptions>true</useStandardDocletOptions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDFS-13833. Improve
BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.
Posted by xk...@apache.org.
HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb
Branch: refs/heads/HDFS-12943
Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530
Parents: 3929653
Author: Shweta <sh...@cloudera.com>
Authored: Tue Sep 18 20:22:25 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 18 20:23:50 2018 -0700
----------------------------------------------------------------------
.../BlockPlacementPolicyDefault.java | 29 ++++++++++++++------
.../blockmanagement/TestReplicationPolicy.java | 28 +++++++++++++++++++
2 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d00f961..d396845 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
/**
+ * Determine if a datanode should be chosen based on current workload.
+ *
+ * @param node The target datanode
+ * @return Return true if the datanode should be excluded, otherwise false
+ */
+ boolean excludeNodeByLoad(DatanodeDescriptor node){
+ final double maxLoad = considerLoadFactor *
+ stats.getInServiceXceiverAverage();
+ final int nodeLoad = node.getXceiverCount();
+ if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
+ logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
+ "(load: " + nodeLoad + " > " + maxLoad + ")");
+ return true;
+ }
+ return false;
+ }
+
+ /**
* Determine if a datanode is good for placing block.
*
* @param node The target datanode
@@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
* @param results A list containing currently chosen nodes. Used to check if
* too many nodes has been chosen in the target rack.
* @param avoidStaleNodes Whether or not to avoid choosing stale nodes
- * @return Reture true if the datanode is good candidate, otherwise false
+ * @return Return true if the datanode is good candidate, otherwise false
*/
boolean isGoodDatanode(DatanodeDescriptor node,
int maxTargetPerRack, boolean considerLoad,
@@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
// check the communication traffic of the target machine
- if (considerLoad) {
- final double maxLoad = considerLoadFactor *
- stats.getInServiceXceiverAverage();
- final int nodeLoad = node.getXceiverCount();
- if (nodeLoad > maxLoad) {
- logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
- "(load: " + nodeLoad + " > " + maxLoad + ")");
+ if(considerLoad){
+ if(excludeNodeByLoad(node)){
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 27dcbf1..f08fa13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
@@ -1559,4 +1560,31 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
}
assertTrue(found);
}
+
+ @Test
+ public void testMaxLoad() {
+ FSClusterStats statistics = mock(FSClusterStats.class);
+ DatanodeDescriptor node = mock(DatanodeDescriptor.class);
+
+ when(statistics.getInServiceXceiverAverage()).thenReturn(0.0);
+ when(node.getXceiverCount()).thenReturn(1);
+
+ final Configuration conf = new Configuration();
+ final Class<? extends BlockPlacementPolicy> replicatorClass = conf
+ .getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
+ BlockPlacementPolicy.class);
+ BlockPlacementPolicy bpp = ReflectionUtils.
+ newInstance(replicatorClass, conf);
+ assertTrue(bpp instanceof BlockPlacementPolicyDefault);
+
+ BlockPlacementPolicyDefault bppd = (BlockPlacementPolicyDefault) bpp;
+ bppd.initialize(conf, statistics, null, null);
+ assertFalse(bppd.excludeNodeByLoad(node));
+
+ when(statistics.getInServiceXceiverAverage()).thenReturn(1.0);
+ when(node.getXceiverCount()).thenReturn(10);
+ assertTrue(bppd.excludeNodeByLoad(node));
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-8784.
DockerLinuxContainerRuntime prevents access to distributed cache entries on a
full disk. Contributed by Eric Badger
Posted by xk...@apache.org.
YARN-8784. DockerLinuxContainerRuntime prevents access to distributed cache entries on a full disk. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b5838ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b5838ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b5838ed
Branch: refs/heads/HDFS-12943
Commit: 6b5838ed3220f992092c7348f92f1d9d0d4a3061
Parents: f6bb1ca
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 19 16:44:51 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 19 16:44:51 2018 -0500
----------------------------------------------------------------------
.../launcher/ContainerLaunch.java | 5 +-
.../launcher/TestContainerLaunch.java | 81 ++++++++++++++++++++
2 files changed, 84 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b5838ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 2aca5f8..6347d4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -250,12 +250,13 @@ public class ContainerLaunch implements Callable<Integer> {
// accessible by users
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
List<String> localDirs = dirsHandler.getLocalDirs();
+ List<String> localDirsForRead = dirsHandler.getLocalDirsForRead();
List<String> logDirs = dirsHandler.getLogDirs();
- List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
+ List<String> filecacheDirs = getNMFilecacheDirs(localDirsForRead);
List<String> userLocalDirs = getUserLocalDirs(localDirs);
List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
List<String> containerLogDirs = getContainerLogDirs(logDirs);
- List<String> userFilecacheDirs = getUserFilecacheDirs(localDirs);
+ List<String> userFilecacheDirs = getUserFilecacheDirs(localDirsForRead);
List<String> applicationLocalDirs = getApplicationLocalDirs(localDirs,
appIdStr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b5838ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index ddf46a6..5714a1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -107,6 +108,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -121,6 +123,7 @@ import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
public class TestContainerLaunch extends BaseContainerManagerTest {
@@ -2362,4 +2365,82 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
}
}
+
+ @Test
+ public void testDistributedCacheDirs() throws Exception {
+ Container container = mock(Container.class);
+ ApplicationId appId =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ ContainerId containerId = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 1);
+ when(container.getContainerId()).thenReturn(containerId);
+ when(container.getUser()).thenReturn("test");
+
+ when(container.getLocalizedResources())
+ .thenReturn(Collections.<Path, List<String>> emptyMap());
+ Dispatcher dispatcher = mock(Dispatcher.class);
+
+ ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+ when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
+ when(container.getLaunchContext()).thenReturn(clc);
+
+ @SuppressWarnings("rawtypes")
+ ContainerExitHandler eventHandler =
+ mock(ContainerExitHandler.class);
+ when(dispatcher.getEventHandler()).thenReturn(eventHandler);
+
+ Application app = mock(Application.class);
+ when(app.getAppId()).thenReturn(appId);
+ when(app.getUser()).thenReturn("test");
+
+ Credentials creds = mock(Credentials.class);
+ when(container.getCredentials()).thenReturn(creds);
+
+ ((NMContext) context).setNodeId(NodeId.newInstance("127.0.0.1", HTTP_PORT));
+ ContainerExecutor mockExecutor = mock(ContainerExecutor.class);
+
+ LocalDirsHandlerService mockDirsHandler =
+ mock(LocalDirsHandlerService.class);
+
+ List <String> localDirsForRead = new ArrayList<String>();
+ String localDir1 =
+ new File("target", this.getClass().getSimpleName() + "-localDir1")
+ .getAbsoluteFile().toString();
+ String localDir2 =
+ new File("target", this.getClass().getSimpleName() + "-localDir2")
+ .getAbsoluteFile().toString();
+ localDirsForRead.add(localDir1);
+ localDirsForRead.add(localDir2);
+
+ List <String> localDirs = new ArrayList();
+ localDirs.add(localDir1);
+ Path logPathForWrite = new Path(localDirs.get(0));
+
+ when(mockDirsHandler.areDisksHealthy()).thenReturn(true);
+ when(mockDirsHandler.getLocalDirsForRead()).thenReturn(localDirsForRead);
+ when(mockDirsHandler.getLocalDirs()).thenReturn(localDirs);
+ when(mockDirsHandler.getLogDirs()).thenReturn(localDirs);
+ when(mockDirsHandler.getLogPathForWrite(anyString(),
+ anyBoolean())).thenReturn(logPathForWrite);
+ when(mockDirsHandler.getLocalPathForWrite(anyString()))
+ .thenReturn(logPathForWrite);
+ when(mockDirsHandler.getLocalPathForWrite(anyString(), anyLong(),
+ anyBoolean())).thenReturn(logPathForWrite);
+
+ ContainerLaunch launch = new ContainerLaunch(context, conf, dispatcher,
+ mockExecutor, app, container, mockDirsHandler, containerManager);
+ launch.call();
+
+ ArgumentCaptor <ContainerStartContext> ctxCaptor =
+ ArgumentCaptor.forClass(ContainerStartContext.class);
+ verify(mockExecutor, times(1)).launchContainer(ctxCaptor.capture());
+ ContainerStartContext ctx = ctxCaptor.getValue();
+
+ Assert.assertEquals(StringUtils.join(",",
+ launch.getNMFilecacheDirs(localDirsForRead)),
+ StringUtils.join(",", ctx.getFilecacheDirs()));
+ Assert.assertEquals(StringUtils.join(",",
+ launch.getUserFilecacheDirs(localDirsForRead)),
+ StringUtils.join(",", ctx.getUserFilecacheDirs()));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8726. [UI2] YARN UI2 is not
accessible when config.env file failed to load. Contributed by Akhil PB.
Posted by xk...@apache.org.
YARN-8726. [UI2] YARN UI2 is not accessible when config.env file failed to load. Contributed by Akhil PB.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cc6e039
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cc6e039
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cc6e039
Branch: refs/heads/HDFS-12943
Commit: 0cc6e039454127984a3aa5b2ba5d9151e4a72dd4
Parents: 51fda2d
Author: Sunil G <su...@apache.org>
Authored: Tue Sep 18 12:31:55 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Sep 18 12:31:55 2018 +0530
----------------------------------------------------------------------
.../hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cc6e039/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
index 96c8fcf..98d1348 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
@@ -88,6 +88,9 @@ function updateConfigs(application) {
var hostname = window.location.hostname;
var rmhost = hostname + (window.location.port ? ':' + window.location.port: '') + skipTrailingSlash(window.location.pathname);
+ window.ENV = window.ENV || {};
+ window.ENV.hosts = window.ENV.hosts || {};
+
if(!ENV.hosts.rmWebAddress) {
ENV.hosts.rmWebAddress = rmhost;
ENV.hosts.protocolScheme = window.location.protocol;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-11719. Arrays.fill() wrong
index in BlockSender.readChecksum() exception handling. Contributed by Tao
Zhang
Posted by xk...@apache.org.
HDFS-11719. Arrays.fill() wrong index in BlockSender.readChecksum() exception handling. Contributed by Tao Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ff509c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ff509c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ff509c3
Branch: refs/heads/HDFS-12943
Commit: 6ff509c32a781eede0e246a0593f6e753fb25d05
Parents: 48319d6
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Sep 18 10:20:15 2018 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Sep 18 10:25:28 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/datanode/BlockSender.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff509c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 268007f..bff47fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -682,16 +682,17 @@ class BlockSender implements java.io.Closeable {
+ " at offset " + offset + " for block " + block, e);
ris.closeChecksumStream();
if (corruptChecksumOk) {
- if (checksumOffset < checksumLen) {
+ if (checksumLen > 0) {
// Just fill the array with zeros.
- Arrays.fill(buf, checksumOffset, checksumLen, (byte) 0);
+ Arrays.fill(buf, checksumOffset, checksumOffset + checksumLen,
+ (byte) 0);
}
} else {
throw e;
}
}
}
-
+
/**
* Compute checksum for chunks and verify the checksum that is read from
* the metadata file is correct.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-8648. Container cgroups are
leaked when using docker. Contributed by Jim Brennan
Posted by xk...@apache.org.
YARN-8648. Container cgroups are leaked when using docker. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2df0a8dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2df0a8dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2df0a8dc
Branch: refs/heads/HDFS-12943
Commit: 2df0a8dcb3dfde15d216481cc1296d97d2cb5d43
Parents: 295cce3
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 15:28:04 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 15:36:45 2018 -0500
----------------------------------------------------------------------
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../linux/resources/ResourceHandlerModule.java | 15 ++
.../runtime/DockerLinuxContainerRuntime.java | 3 +-
.../linux/runtime/docker/DockerRmCommand.java | 11 +-
.../impl/container-executor.c | 153 ++++++++++++++++++-
.../impl/container-executor.h | 8 +-
.../main/native/container-executor/impl/main.c | 12 +-
.../test/test-container-executor.c | 147 ++++++++++++++++++
.../docker/TestDockerCommandExecutor.java | 23 ++-
.../runtime/docker/TestDockerRmCommand.java | 35 ++++-
10 files changed, 393 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index b3c9d5f..fccf668 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -937,7 +937,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
DockerCommandExecutor.getContainerStatus(containerId, privOpExecutor,
nmContext))) {
LOG.info("Removing Docker container : " + containerId);
- DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId);
+ DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId,
+ ResourceHandlerModule.getCgroupsRelativeRoot());
DockerCommandExecutor.executeDockerCommand(dockerRmCommand, containerId,
null, privOpExecutor, false, nmContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
index fc55696..f8a3193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
@@ -101,6 +101,21 @@ public class ResourceHandlerModule {
return cGroupsHandler;
}
+ /**
+ * Returns relative root for cgroups. Returns null if cGroupsHandler is
+ * not initialized, or if the path is empty.
+ */
+ public static String getCgroupsRelativeRoot() {
+ if (cGroupsHandler == null) {
+ return null;
+ }
+ String cGroupPath = cGroupsHandler.getRelativePathForCGroup("");
+ if (cGroupPath == null || cGroupPath.isEmpty()) {
+ return null;
+ }
+ return cGroupPath.replaceAll("/$", "");
+ }
+
public static NetworkPacketTaggingHandlerImpl
getNetworkResourceHandler() {
return networkPacketTaggingHandlerImpl;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 8b2b404..2b53f13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1382,7 +1382,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
DockerCommandExecutor.getContainerStatus(containerId,
privilegedOperationExecutor, nmContext);
if (DockerCommandExecutor.isRemovable(containerStatus)) {
- DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId);
+ DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId,
+ ResourceHandlerModule.getCgroupsRelativeRoot());
DockerCommandExecutor.executeDockerCommand(dockerRmCommand, containerId,
env, privilegedOperationExecutor, false, nmContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
index 490cf9e..b4b692b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
@@ -27,10 +27,16 @@ import java.util.Map;
*/
public class DockerRmCommand extends DockerCommand {
private static final String RM_COMMAND = "rm";
+ private static final String CGROUP_HIERARCHY = "hierarchy";
+ private String cGroupArg;
- public DockerRmCommand(String containerName) {
+ public DockerRmCommand(String containerName, String hierarchy) {
super(RM_COMMAND);
super.addCommandArguments("name", containerName);
+ if ((hierarchy != null) && !hierarchy.isEmpty()) {
+ super.addCommandArguments(CGROUP_HIERARCHY, hierarchy);
+ this.cGroupArg = hierarchy;
+ }
}
@Override
@@ -39,6 +45,9 @@ public class DockerRmCommand extends DockerCommand {
String> env, Context nmContext) {
PrivilegedOperation dockerOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.REMOVE_DOCKER_CONTAINER);
+ if (this.cGroupArg != null) {
+ dockerOp.appendArgs(cGroupArg);
+ }
dockerOp.appendArgs(containerName);
return dockerOp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index f8b89ee..7765308 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -46,6 +46,7 @@
#include <sys/mount.h>
#include <sys/wait.h>
#include <getopt.h>
+#include <sys/param.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -1374,17 +1375,16 @@ int run_docker(const char *command_file) {
return exit_code;
}
-int exec_docker_command(char *docker_command, char **argv,
- int argc, int optind) {
+int exec_docker_command(char *docker_command, char **argv, int argc) {
int i;
char* docker_binary = get_docker_binary(&CFG);
- size_t command_size = argc - optind + 2;
+ size_t command_size = argc + 2;
- char **args = alloc_and_clear_memory(command_size + 1, sizeof(char));
+ char **args = alloc_and_clear_memory(command_size + 1, sizeof(char *));
args[0] = docker_binary;
args[1] = docker_command;
for(i = 2; i < command_size; i++) {
- args[i] = (char *) argv[i];
+ args[i] = (char *) argv[i - 2];
}
args[i] = NULL;
@@ -2565,4 +2565,147 @@ char* flatten(char **args) {
return buffer;
}
+int clean_docker_cgroups_internal(const char *mount_table,
+ const char *yarn_hierarchy,
+ const char* container_id) {
+#ifndef __linux
+ fprintf(LOGFILE, "Failed to clean cgroups, not supported\n");
+ return -1;
+#else
+ const char * cgroup_mount_type = "cgroup";
+ char *mnt_type = NULL;
+ char *mnt_dir = NULL;
+ char *full_path = NULL;
+ char *lineptr = NULL;
+ FILE *fp = NULL;
+ int rc = 0;
+ size_t buf_size = 0;
+
+ if (!mount_table || mount_table[0] == 0) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Invalid mount table\n");
+ rc = -1;
+ goto cleanup;
+ }
+ if (!yarn_hierarchy || yarn_hierarchy[0] == 0) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Invalid yarn_hierarchy\n");
+ rc = -1;
+ goto cleanup;
+ }
+ if (!validate_container_id(container_id)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Invalid container_id: %s\n",
+ (container_id == NULL) ? "null" : container_id);
+ rc = -1;
+ goto cleanup;
+ }
+ fp = fopen(mount_table, "r");
+ if (fp == NULL) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: failed to open %s, error %d: %s\n",
+ mount_table, errno, strerror(errno));
+ rc = -1;
+ goto cleanup;
+ }
+
+ // Walk /proc/mounts and find cgroup mounts
+ while (getline(&lineptr, &buf_size, fp) != -1) {
+ // Free these from the last iteration, if set
+ free(mnt_type);
+ free(mnt_dir);
+ int ret = 0;
+ ret = sscanf(lineptr, " %ms %ms %*s %*s %*s %*s", &mnt_type, &mnt_dir);
+ if (ret != 2) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Failed to parse line: %s\n", lineptr);
+ rc = -1;
+ break;
+ }
+ if ((mnt_type == NULL) || (strcmp(mnt_type, cgroup_mount_type) != 0)) {
+ continue;
+ }
+ if ((mnt_dir == NULL) || (mnt_dir[0] == 0)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: skipping mount entry with invalid mnt_dir\n");
+ continue;
+ }
+
+ free(full_path); // from previous iteration
+ full_path = make_string("%s/%s/%s", mnt_dir, yarn_hierarchy, container_id);
+ if (full_path == NULL) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Failed to allocate cgroup path.\n");
+ rc = -1;
+ break;
+ }
+
+ // Make sure path is clean
+ if (!verify_path_safety(full_path)) {
+ fprintf(ERRORFILE,
+ "clean_docker_cgroups: skipping invalid path: %s\n", full_path);
+ continue;
+ }
+
+ ret = rmdir(full_path);
+ if ((ret == -1) && (errno != ENOENT)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Failed to rmdir cgroup, %s (error=%s)\n",
+ full_path, strerror(errno));
+ rc = -1;
+ continue;
+ }
+ }
+ if (ferror(fp)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Error reading %s, error=%d (%s) \n",
+ mount_table, errno, strerror(errno));
+ rc = -1;
+ }
+
+cleanup:
+ free(lineptr);
+ free(mnt_type);
+ free(mnt_dir);
+ free(full_path);
+ if (fp != NULL) {
+ fclose(fp);
+ }
+ return rc;
+#endif
+}
+
+int clean_docker_cgroups(const char *yarn_hierarchy, const char* container_id) {
+ const char *proc_mount_path = "/proc/mounts";
+ return clean_docker_cgroups_internal(proc_mount_path, yarn_hierarchy, container_id);
+}
+
+int remove_docker_container(char**argv, int argc) {
+ int exit_code = 0;
+ const char *yarn_hierarchy = NULL;
+ const char *container_id = NULL;
+ int start_index = 0;
+ if (argc == 2) {
+ yarn_hierarchy = argv[0];
+ container_id = argv[1];
+ // Skip the yarn_hierarchy argument for exec_docker_command
+ start_index = 1;
+ }
+
+ pid_t child_pid = fork();
+ if (child_pid == -1) {
+ fprintf (ERRORFILE,
+ "Failed to fork for docker remove command\n");
+ fflush(ERRORFILE);
+ return DOCKER_RUN_FAILED;
+ }
+
+ if (child_pid == 0) { // child
+ int rc = exec_docker_command("rm", argv + start_index, argc - start_index);
+ return rc; // Only get here if exec fails
+
+ } else { // parent
+ exit_code = wait_and_get_exit_code(child_pid);
+ if (exit_code != 0) {
+ exit_code = DOCKER_RUN_FAILED;
+ }
+ }
+
+ // Clean up cgroups if necessary
+ if (yarn_hierarchy != NULL) {
+ exit_code = clean_docker_cgroups(yarn_hierarchy, container_id);
+ }
+ return exit_code;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 002f85f..cd09e1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -271,8 +271,7 @@ int run_docker(const char *command_file);
/**
* Run a docker command without a command file
*/
-int exec_docker_command(char *docker_command, char **argv,
- int argc, int optind);
+int exec_docker_command(char *docker_command, char **argv, int argc);
/*
* Compile the regex_str and determine if the input string matches.
@@ -292,3 +291,8 @@ struct configuration* get_cfg();
* Flatten docker launch command
*/
char* flatten(char **args);
+
+/**
+ * Remove docker container
+ */
+int remove_docker_container(char **argv, int argc);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 93691f9..a3057e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -54,12 +54,14 @@ static void display_usage(FILE *stream) {
if(is_docker_support_enabled()) {
fprintf(stream,
" container-executor --run-docker <command-file>\n"
- " container-executor --remove-docker-container <container_id>\n"
+ " container-executor --remove-docker-container [hierarchy] "
+ "<container_id>\n"
" container-executor --inspect-docker-container <container_id>\n");
} else {
fprintf(stream,
"[DISABLED] container-executor --run-docker <command-file>\n"
- "[DISABLED] container-executor --remove-docker-container <container_id>\n"
+ "[DISABLED] container-executor --remove-docker-container [hierarchy] "
+ "<container_id>\n"
"[DISABLED] container-executor --inspect-docker-container "
"<format> ... <container_id>\n");
}
@@ -351,7 +353,7 @@ static int validate_arguments(int argc, char **argv , int *operation) {
if (strcmp("--remove-docker-container", argv[1]) == 0) {
if(is_docker_support_enabled()) {
- if (argc != 3) {
+ if ((argc != 3) && (argc != 4)) {
display_usage(stdout);
return INVALID_ARGUMENT_NUMBER;
}
@@ -594,10 +596,10 @@ int main(int argc, char **argv) {
exit_code = run_docker(cmd_input.docker_command_file);
break;
case REMOVE_DOCKER_CONTAINER:
- exit_code = exec_docker_command("rm", argv, argc, optind);
+ exit_code = remove_docker_container(argv + optind, argc - optind);
break;
case INSPECT_DOCKER_CONTAINER:
- exit_code = exec_docker_command("inspect", argv, argc, optind);
+ exit_code = exec_docker_command("inspect", argv + optind, argc - optind);
break;
case RUN_AS_USER_INITIALIZE_CONTAINER:
exit_code = set_user(cmd_input.run_as_user_name);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 5607823..f4f00c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -23,6 +23,7 @@
#include "test/test-container-executor-common.h"
#include <inttypes.h>
+#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
@@ -32,6 +33,8 @@
#include <string.h>
#include <sys/stat.h>
#include <sys/wait.h>
+#include <sys/param.h>
+
static char* username = NULL;
static char* yarn_username = NULL;
@@ -1224,6 +1227,148 @@ void test_is_empty() {
}
}
+#define TCE_FAKE_CGROOT TEST_ROOT "/cgroup_root"
+#define TCE_NUM_CG_CONTROLLERS 6
+extern int clean_docker_cgroups_internal(const char *mount_table,
+ const char *yarn_hierarchy,
+ const char* container_id);
+
+void test_cleaning_docker_cgroups() {
+ const char *controllers[TCE_NUM_CG_CONTROLLERS] = { "blkio", "cpu", "cpuset", "devices", "memory", "systemd" };
+ const char *yarn_hierarchy = "hadoop-yarn";
+ const char *fake_mount_table = TEST_ROOT "/fake_mounts";
+ const char *container_id = "container_1410901177871_0001_01_000005";
+ const char *other_container_id = "container_e17_1410901177871_0001_01_000005";
+ char cgroup_paths[TCE_NUM_CG_CONTROLLERS][PATH_MAX];
+ char container_paths[TCE_NUM_CG_CONTROLLERS][PATH_MAX];
+ char other_container_paths[TCE_NUM_CG_CONTROLLERS][PATH_MAX];
+
+ printf("\nTesting clean_docker_cgroups\n");
+
+ // Setup fake mount table
+ FILE *file;
+ file = fopen(fake_mount_table, "w");
+ if (file == NULL) {
+ printf("Failed to open %s.\n", fake_mount_table);
+ exit(1);
+ }
+ fprintf(file, "rootfs " TEST_ROOT "/fake_root rootfs rw 0 0\n");
+ fprintf(file, "sysfs " TEST_ROOT "/fake_sys sysfs rw,nosuid,nodev,noexec,relatime 0 0\n");
+ fprintf(file, "proc " TEST_ROOT "/fake_proc proc rw,nosuid,nodev,noexec,relatime 0 0\n");
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ fprintf(file, "cgroup %s/%s cgroup rw,nosuid,nodev,noexec,relatime,%s 0 0\n",
+ TCE_FAKE_CGROOT, controllers[i], controllers[i]);
+ }
+ fprintf(file, "/dev/vda " TEST_ROOT "/fake_root ext4 rw,relatime,data=ordered 0 0\n");
+ fclose(file);
+
+ // Test with null inputs
+ int ret = clean_docker_cgroups_internal(NULL, yarn_hierarchy, container_id);
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with NULL mount table should fail\n");
+ exit(1);
+ }
+ ret = clean_docker_cgroups_internal(fake_mount_table, NULL, container_id);
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with NULL yarn_hierarchy should fail\n");
+ exit(1);
+ }
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, NULL);
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with NULL container_id should fail\n");
+ exit(1);
+ }
+
+ // Test with invalid container_id
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, "not_a_container_123");
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with invalid container_id should fail\n");
+ exit(1);
+ }
+ if (mkdir(TCE_FAKE_CGROOT, 0755) != 0) {
+ printf("FAIL: failed to mkdir " TCE_FAKE_CGROOT "\n");
+ exit(1);
+ }
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ snprintf(cgroup_paths[i], PATH_MAX, TCE_FAKE_CGROOT "/%s/%s", controllers[i], yarn_hierarchy);
+ if (mkdirs(cgroup_paths[i], 0755) != 0) {
+ printf("FAIL: failed to mkdir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ }
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(cgroup_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: failed to open dir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // Test before creating any containers
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, container_id);
+ if (ret != 0) {
+ printf("FAIL: failed to clean cgroups: mt=%s, yh=%s, cId=%s\n",
+ fake_mount_table, yarn_hierarchy, container_id);
+ }
+ // make sure hadoop-yarn dirs are still there
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(cgroup_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: failed to open dir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // Create container dirs
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ snprintf(container_paths[i], PATH_MAX, TCE_FAKE_CGROOT "/%s/%s/%s",
+ controllers[i], yarn_hierarchy, container_id);
+ if (mkdirs(container_paths[i], 0755) != 0) {
+ printf("FAIL: failed to mkdir %s\n", container_paths[i]);
+ exit(1);
+ }
+ snprintf(other_container_paths[i], PATH_MAX, TCE_FAKE_CGROOT "/%s/%s/%s",
+ controllers[i], yarn_hierarchy, other_container_id);
+ if (mkdirs(other_container_paths[i], 0755) != 0) {
+ printf("FAIL: failed to mkdir %s\n", other_container_paths[i]);
+ exit(1);
+ }
+ }
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, container_id);
+ // make sure hadoop-yarn dirs are still there
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(cgroup_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: failed to open dir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // make sure container dirs deleted
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(container_paths[i]);
+ if (dir != NULL) {
+ printf("FAIL: container cgroup %s not deleted\n", container_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // make sure other container dirs are still there
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(other_container_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: container cgroup %s should not be deleted\n", other_container_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+}
+
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1328,6 +1473,8 @@ int main(int argc, char **argv) {
test_check_user(0);
+ test_cleaning_docker_cgroups();
+
#ifdef __APPLE__
printf("OS X: disabling CrashReporter\n");
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
index 46415c1..143e9b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
@@ -66,6 +66,7 @@ public class TestDockerCommandExecutor {
"container_e11_1861047502093_13763105_01_000001";
private static final String MOCK_LOCAL_IMAGE_NAME = "local_image_name";
private static final String MOCK_IMAGE_NAME = "image_name";
+ private static final String MOCK_CGROUP_HIERARCHY = "hadoop-yarn";
private PrivilegedOperationExecutor mockExecutor;
private CGroupsHandler mockCGroupsHandler;
@@ -148,7 +149,8 @@ public class TestDockerCommandExecutor {
@Test
public void testExecuteDockerRm() throws Exception {
- DockerRmCommand dockerCommand = new DockerRmCommand(MOCK_CONTAINER_ID);
+ DockerRmCommand dockerCommand =
+ new DockerRmCommand(MOCK_CONTAINER_ID, null);
DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
env, mockExecutor, false, nmContext);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
@@ -164,6 +166,25 @@ public class TestDockerCommandExecutor {
}
@Test
+ public void testExecuteDockerRmWithCgroup() throws Exception {
+ DockerRmCommand dockerCommand =
+ new DockerRmCommand(MOCK_CONTAINER_ID, MOCK_CGROUP_HIERARCHY);
+ DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
+ env, mockExecutor, false, nmContext);
+ List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
+ .capturePrivilegedOperations(mockExecutor, 1, true);
+ PrivilegedOperation privOp = ops.get(0);
+ List<String> args = privOp.getArguments();
+ assertEquals(1, ops.size());
+ assertEquals(PrivilegedOperation.OperationType.
+ REMOVE_DOCKER_CONTAINER.name(),
+ privOp.getOperationType().name());
+ assertEquals(2, args.size());
+ assertEquals(MOCK_CGROUP_HIERARCHY, args.get(0));
+ assertEquals(MOCK_CONTAINER_ID, args.get(1));
+ }
+
+ @Test
public void testExecuteDockerStop() throws Exception {
DockerStopCommand dockerCommand = new DockerStopCommand(MOCK_CONTAINER_ID);
DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
index a8d4bdd..8a7c876 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
@@ -29,12 +29,19 @@ import org.junit.Test;
public class TestDockerRmCommand {
private DockerRmCommand dockerRmCommand;
+ private DockerRmCommand dockerRmCommandWithCgroupArg;
+ private DockerRmCommand dockerRmCommandWithEmptyCgroupArg;
private static final String CONTAINER_NAME = "foo";
+ private static final String CGROUP_HIERARCHY_NAME = "hadoop-yarn";
@Before
public void setUp() {
- dockerRmCommand = new DockerRmCommand(CONTAINER_NAME);
+ dockerRmCommand = new DockerRmCommand(CONTAINER_NAME, null);
+ dockerRmCommandWithCgroupArg =
+ new DockerRmCommand(CONTAINER_NAME, CGROUP_HIERARCHY_NAME);
+ dockerRmCommandWithEmptyCgroupArg =
+ new DockerRmCommand(CONTAINER_NAME, "");
}
@Test
@@ -51,4 +58,30 @@ public class TestDockerRmCommand {
assertEquals(2, dockerRmCommand.getDockerCommandWithArguments().size());
}
+ @Test
+ public void testGetCommandWithCgroup() {
+ assertEquals("rm", StringUtils.join(",",
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments()
+ .get("name")));
+ assertEquals(CGROUP_HIERARCHY_NAME, StringUtils.join(",",
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments()
+ .get("hierarchy")));
+ assertEquals(3,
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments().size());
+ }
+
+ @Test
+ public void testGetCommandWithEmptyCgroup() {
+ assertEquals("rm", StringUtils.join(",",
+ dockerRmCommandWithEmptyCgroupArg
+ .getDockerCommandWithArguments().get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRmCommandWithEmptyCgroupArg
+ .getDockerCommandWithArguments().get("name")));
+ assertEquals(2, dockerRmCommandWithEmptyCgroupArg.
+ getDockerCommandWithArguments().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HADOOP-15684. triggerActiveLogRoll
stuck on dead name node,
when ConnectTimeoutException happens. Contributed by Rong Tang.
Posted by xk...@apache.org.
HADOOP-15684. triggerActiveLogRoll stuck on dead name node, when ConnectTimeoutException happens. Contributed by Rong Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/090272d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/090272d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/090272d7
Branch: refs/heads/HDFS-12943
Commit: 090272d7de7cb5f1133359d66780aef7c5cce5c9
Parents: 236d16e
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Sep 19 12:58:31 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Sep 19 13:00:30 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/ha/EditLogTailer.java | 34 ++-----
.../apache/hadoop/hdfs/MiniDFSNNTopology.java | 17 ++++
.../server/namenode/ha/TestEditLogTailer.java | 98 +++++++++++++-------
3 files changed, 93 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 4ba2aa3..fc5f3a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -53,8 +53,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.SecurityUtil;
import static org.apache.hadoop.util.Time.monotonicNow;
@@ -382,15 +380,6 @@ public class EditLogTailer {
future.get(rollEditsTimeoutMs, TimeUnit.MILLISECONDS);
lastRollTriggerTxId = lastLoadedTxnId;
} catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof RemoteException) {
- IOException ioe = ((RemoteException) cause).unwrapRemoteException();
- if (ioe instanceof StandbyException) {
- LOG.info("Skipping log roll. Remote node is not in Active state: " +
- ioe.getMessage().split("\n")[0]);
- return;
- }
- }
LOG.warn("Unable to trigger a roll of the active NN", e);
} catch (TimeoutException e) {
if (future != null) {
@@ -497,7 +486,8 @@ public class EditLogTailer {
* This mechanism is <b>very bad</b> for cases where we care about being <i>fast</i>; it just
* blindly goes and tries namenodes.
*/
- private abstract class MultipleNameNodeProxy<T> implements Callable<T> {
+ @VisibleForTesting
+ abstract class MultipleNameNodeProxy<T> implements Callable<T> {
/**
* Do the actual work to the remote namenode via the {@link #cachedActiveProxy}.
@@ -513,19 +503,13 @@ public class EditLogTailer {
try {
T ret = doWork();
return ret;
- } catch (RemoteException e) {
- Throwable cause = e.unwrapRemoteException(StandbyException.class);
- // if its not a standby exception, then we need to re-throw it, something bad has happened
- if (cause == e) {
- throw e;
- } else {
- // it is a standby exception, so we try the other NN
- LOG.warn("Failed to reach remote node: " + currentNN
- + ", retrying with remaining remote NNs");
- cachedActiveProxy = null;
- // this NN isn't responding to requests, try the next one
- nnLoopCount++;
- }
+ } catch (IOException e) {
+ LOG.warn("Exception from remote name node " + currentNN
+ + ", try next.", e);
+
+ // Try next name node if exception happens.
+ cachedActiveProxy = null;
+ nnLoopCount++;
}
}
throw new IOException("Cannot find any valid remote NN to service request!");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
index c21ff80..390c61c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
@@ -73,6 +73,23 @@ public class MiniDFSNNTopology {
}
/**
+ * Set up an HA topology with a single HA nameservice.
+ * @param nnCount of namenodes to use with the nameservice
+ * @param basePort for IPC and Http ports of namenodes.
+ */
+ public static MiniDFSNNTopology simpleHATopology(int nnCount, int basePort) {
+ MiniDFSNNTopology.NSConf ns = new MiniDFSNNTopology.NSConf("minidfs-ns");
+ for (int i = 0; i < nnCount; i++) {
+ ns.addNN(new MiniDFSNNTopology.NNConf("nn" + i)
+ .setIpcPort(basePort++)
+ .setHttpPort(basePort++));
+ }
+ MiniDFSNNTopology topology = new MiniDFSNNTopology()
+ .addNameservice(ns);
+ return topology;
+ }
+
+ /**
* Set up federated cluster with the given number of nameservices, each
* of which has only a single NameNode.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
index 68b3e2b..b94cd2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
@@ -28,6 +28,7 @@ import java.net.BindException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -46,7 +47,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.slf4j.event.Level;
import org.junit.Test;
@@ -177,21 +177,7 @@ public class TestEditLogTailer {
MiniDFSCluster cluster = null;
for (int i = 0; i < 5; i++) {
try {
- // Have to specify IPC ports so the NNs can talk to each other.
- int[] ports = ServerSocketUtil.getPorts(3);
- MiniDFSNNTopology topology = new MiniDFSNNTopology()
- .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
- .addNN(new MiniDFSNNTopology.NNConf("nn1")
- .setIpcPort(ports[0]))
- .addNN(new MiniDFSNNTopology.NNConf("nn2")
- .setIpcPort(ports[1]))
- .addNN(new MiniDFSNNTopology.NNConf("nn3")
- .setIpcPort(ports[2])));
-
- cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .build();
+ cluster = createMiniDFSCluster(conf, 3);
break;
} catch (BindException e) {
// retry if race on ports given by ServerSocketUtil#getPorts
@@ -222,21 +208,9 @@ public class TestEditLogTailer {
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY, 100);
- // Have to specify IPC ports so the NNs can talk to each other.
- MiniDFSNNTopology topology = new MiniDFSNNTopology()
- .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
- .addNN(new MiniDFSNNTopology.NNConf("nn1")
- .setIpcPort(ServerSocketUtil.getPort(0, 100)))
- .addNN(new MiniDFSNNTopology.NNConf("nn2")
- .setIpcPort(ServerSocketUtil.getPort(0, 100)))
- .addNN(new MiniDFSNNTopology.NNConf("nn3")
- .setIpcPort(ServerSocketUtil.getPort(0, 100))));
-
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .build();
+ MiniDFSCluster cluster = null;
try {
+ cluster = createMiniDFSCluster(conf, 3);
cluster.transitionToStandby(0);
cluster.transitionToStandby(1);
cluster.transitionToStandby(2);
@@ -249,7 +223,9 @@ public class TestEditLogTailer {
cluster.transitionToActive(0);
waitForLogRollInSharedDir(cluster, 3);
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -316,4 +292,64 @@ public class TestEditLogTailer {
cluster.shutdown();
}
}
+
+ @Test
+ public void testRollEditLogIOExceptionForRemoteNN() throws IOException {
+ Configuration conf = getConf();
+
+ // Roll every 1s
+ conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = createMiniDFSCluster(conf, 3);
+ cluster.transitionToActive(0);
+ EditLogTailer tailer = Mockito.spy(
+ cluster.getNamesystem(1).getEditLogTailer());
+
+ final AtomicInteger invokedTimes = new AtomicInteger(0);
+
+ // It should go on to next name node when IOException happens.
+ when(tailer.getNameNodeProxy()).thenReturn(
+ tailer.new MultipleNameNodeProxy<Void>() {
+ @Override
+ protected Void doWork() throws IOException {
+ invokedTimes.getAndIncrement();
+ throw new IOException("It is an IO Exception.");
+ }
+ }
+ );
+
+ tailer.triggerActiveLogRoll();
+
+ // MultipleNameNodeProxy uses Round-robin to look for active NN
+ // to do RollEditLog. If doWork() fails, then IOException throws,
+ // it continues to try next NN. triggerActiveLogRoll finishes
+ // either due to success, or using up retries.
+ // In this test case, there are 2 remote name nodes, default retry is 3.
+ // For test purpose, doWork() always returns IOException,
+ // so the total invoked times will be default retry 3 * remote NNs 2 = 6
+ assertEquals(6, invokedTimes.get());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private static MiniDFSCluster createMiniDFSCluster(Configuration conf,
+ int nnCount) throws IOException {
+ int basePort = 10060 + new Random().nextInt(100) * 2;
+
+ // By passing in basePort, name node will have IPC port set,
+ // which is needed for enabling roll log.
+ MiniDFSNNTopology topology =
+ MiniDFSNNTopology.simpleHATopology(nnCount, basePort);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(topology)
+ .numDataNodes(0)
+ .build();
+ return cluster;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org