You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/11/08 18:11:13 UTC
[01/29] hadoop git commit: HDFS-14043. Tolerate corrupted seen_txid
file. Contributed by Lukas Majercak.
Repository: hadoop
Updated Branches:
refs/heads/HDDS-4 10d395453 -> 629347bea
HDFS-14043. Tolerate corrupted seen_txid file. Contributed by Lukas Majercak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3296501
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3296501
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3296501
Branch: refs/heads/HDDS-4
Commit: f3296501e09fa7f1e81548dfcefa56f20fe337ca
Parents: f3f5e7a
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Nov 5 16:48:37 2018 -0800
Committer: Inigo Goiri <in...@apache.org>
Committed: Mon Nov 5 16:48:37 2018 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/util/PersistentLongFile.java | 2 +
.../hdfs/server/namenode/TestSaveNamespace.java | 56 ++++++++++++++++++++
2 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3296501/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
index 777dd87..a94d7ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
@@ -98,6 +98,8 @@ public class PersistentLongFile {
val = Long.parseLong(br.readLine());
br.close();
br = null;
+ } catch (NumberFormatException e) {
+ throw new IOException(e);
} finally {
IOUtils.cleanupWithLogger(LOG, br);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3296501/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index 8fa8701..6688ef2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -28,13 +28,20 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -737,6 +744,55 @@ public class TestSaveNamespace {
}
}
+ @Test(timeout=30000)
+ public void testTxFaultTolerance() throws Exception {
+ String baseDir = MiniDFSCluster.getBaseDirectory();
+ List<String> nameDirs = new ArrayList<>();
+ nameDirs.add(fileAsURI(new File(baseDir, "name1")).toString());
+ nameDirs.add(fileAsURI(new File(baseDir, "name2")).toString());
+
+ Configuration conf = new HdfsConfiguration();
+ String nameDirsStr = StringUtils.join(",", nameDirs);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameDirsStr);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameDirsStr);
+
+ NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
+ DFSTestUtil.formatNameNode(conf);
+ FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
+ try {
+ // We have a BEGIN_LOG_SEGMENT txn to start
+ assertEquals(1, fsn.getEditLog().getLastWrittenTxId());
+
+ doAnEdit(fsn, 1);
+
+ assertEquals(2, fsn.getEditLog().getLastWrittenTxId());
+
+ // Shut down
+ fsn.close();
+
+ // Corrupt one of the seen_txid files
+ File txidFile0 = new File(new URI(nameDirs.get(0) +
+ "/current/seen_txid"));
+ FileWriter fw = new FileWriter(txidFile0, false);
+ try (PrintWriter pw = new PrintWriter(fw)) {
+ pw.print("corrupt____!");
+ }
+
+ // Restart
+ fsn = FSNamesystem.loadFromDisk(conf);
+ assertEquals(4, fsn.getEditLog().getLastWrittenTxId());
+
+ // Check seen_txid is same in both dirs
+ File txidFile1 = new File(new URI(nameDirs.get(1) +
+ "/current/seen_txid"));
+ assertTrue(FileUtils.contentEquals(txidFile0, txidFile1));
+ } finally {
+ if (fsn != null) {
+ fsn.close();
+ }
+ }
+ }
+
private void doAnEdit(FSNamesystem fsn, int id) throws IOException {
// Make an edit
fsn.mkdirs("/test" + id, new PermissionStatus("test", "Test",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/29] hadoop git commit: MAPREDUCE-7148. Fast fail jobs when
exceeds dfs quota limitation. Contributed by Wang Yan
Posted by ae...@apache.org.
MAPREDUCE-7148. Fast fail jobs when exceeds dfs quota limitation. Contributed by Wang Yan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b6625a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b6625a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b6625a9
Branch: refs/heads/HDDS-4
Commit: 0b6625a9735f76ab473b41d8ab9b7f3c7678cfff
Parents: 8dc1f6d
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Nov 7 08:20:49 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Nov 7 08:20:49 2018 -0600
----------------------------------------------------------------------
...ClusterStorageCapacityExceededException.java | 51 ++++++++
.../hdfs/protocol/QuotaExceededException.java | 5 +-
.../org/apache/hadoop/mapred/YarnChild.java | 28 ++++-
.../org/apache/hadoop/mapred/TestYarnChild.java | 118 +++++++++++++++++++
.../apache/hadoop/mapreduce/MRJobConfig.java | 3 +
.../src/main/resources/mapred-default.xml | 9 ++
6 files changed, 209 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b6625a9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java
new file mode 100644
index 0000000..bbbf073
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Exception raised by HDFS indicating that storage capacity in the
+ * cluster filesystem is exceeded. See also
+ * https://issues.apache.org/jira/browse/MAPREDUCE-7148.
+ */
+@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
+@InterfaceStability.Evolving
+public class ClusterStorageCapacityExceededException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public ClusterStorageCapacityExceededException() {
+ super();
+ }
+
+ public ClusterStorageCapacityExceededException(String message) {
+ super(message);
+ }
+
+ public ClusterStorageCapacityExceededException(String message,
+ Throwable cause) {
+ super(message, cause);
+ }
+
+ public ClusterStorageCapacityExceededException(Throwable cause) {
+ super(cause);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b6625a9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaExceededException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaExceededException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaExceededException.java
index f4e7f34..7033f3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaExceededException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaExceededException.java
@@ -18,10 +18,9 @@
package org.apache.hadoop.hdfs.protocol;
-import java.io.IOException;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ClusterStorageCapacityExceededException;
/**
* This exception is thrown when modification to HDFS results in violation
@@ -37,7 +36,7 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class QuotaExceededException extends IOException {
+public class QuotaExceededException extends ClusterStorageCapacityExceededException {
protected static final long serialVersionUID = 1L;
protected String pathName=null;
protected long quota; // quota
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b6625a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index bd40e54..e81b090 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -26,10 +26,12 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.ScheduledExecutorService;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.ClusterStorageCapacityExceededException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.CallerContext;
@@ -61,6 +63,8 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* The main() for MapReduce task processes.
*/
@@ -205,8 +209,7 @@ class YarnChild {
// Report back any failures, for diagnostic purposes
if (taskid != null) {
if (!ShutdownHookManager.get().isShutdownInProgress()) {
- umbilical.fatalError(taskid,
- StringUtils.stringifyException(exception), false);
+ reportError(exception, task, umbilical);
}
}
} catch (Throwable throwable) {
@@ -228,6 +231,27 @@ class YarnChild {
}
}
+ @VisibleForTesting
+ static void reportError(Exception exception, Task task,
+ TaskUmbilicalProtocol umbilical) throws IOException {
+ boolean fastFailJob = false;
+ boolean hasClusterStorageCapacityExceededException =
+ ExceptionUtils.indexOfType(exception,
+ ClusterStorageCapacityExceededException.class) != -1;
+ if (hasClusterStorageCapacityExceededException) {
+ boolean killJobWhenExceedClusterStorageCapacity = task.getConf()
+ .getBoolean(MRJobConfig.JOB_DFS_STORAGE_CAPACITY_KILL_LIMIT_EXCEED,
+ MRJobConfig.DEFAULT_JOB_DFS_STORAGE_CAPACITY_KILL_LIMIT_EXCEED);
+ if (killJobWhenExceedClusterStorageCapacity) {
+ LOG.error(
+ "Fast fail the job because the cluster storage capacity was exceeded.");
+ fastFailJob = true;
+ }
+ }
+ umbilical.fatalError(taskid, StringUtils.stringifyException(exception),
+ fastFailJob);
+ }
+
/**
* Utility method to check if the Encrypted Spill Key needs to be set into the
* user credentials of the user running the Map / Reduce Task
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b6625a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestYarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestYarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestYarnChild.java
new file mode 100644
index 0000000..404b1cb
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestYarnChild.java
@@ -0,0 +1,118 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ClusterStorageCapacityExceededException;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.mockito.Mockito.*;
+
+/**
+ * Tests the behavior of YarnChild.
+ */
+public class TestYarnChild {
+ private Task task;
+ private TaskUmbilicalProtocol umbilical;
+ private Configuration conf;
+ final static private String KILL_LIMIT_EXCEED_CONF_NAME =
+ "mapreduce.job.dfs.storage.capacity.kill-limit-exceed";
+
+ @Before
+ public void setUp() throws Exception {
+ task = mock(Task.class);
+ umbilical = mock(TaskUmbilicalProtocol.class);
+ conf = new Configuration();
+ when(task.getConf()).thenReturn(conf);
+ }
+
+ @Test
+ public void testReportErrorWhenCapacityExceptionNotHappenByDefault()
+ throws IOException {
+ Exception exception = new RuntimeException(new IOException());
+
+ verifyReportError(exception, false);
+ }
+
+ @Test
+ public void testReportErrorWhenCapacityExceptionNotHappenAndFastFailDisabled()
+ throws IOException {
+ Exception exception = new RuntimeException(new IOException());
+ conf.setBoolean(KILL_LIMIT_EXCEED_CONF_NAME, false);
+
+ verifyReportError(exception, false);
+ }
+
+ @Test
+ public void testReportErrorWhenCapacityExceptionNotHappenAndFastFailEnabled()
+ throws IOException {
+ Exception exception = new RuntimeException(new IOException());
+ conf.setBoolean(KILL_LIMIT_EXCEED_CONF_NAME, true);
+
+ verifyReportError(exception, false);
+ }
+
+ @Test
+ public void testReportErrorWhenCapacityExceptionHappenByDefault()
+ throws IOException {
+ Exception exception =
+ new RuntimeException(new ClusterStorageCapacityExceededException());
+
+ verifyReportError(exception, false);
+ }
+
+ @Test
+ public void testReportErrorWhenCapacityExceptionHappenAndFastFailDisabled()
+ throws IOException {
+ Exception exception =
+ new RuntimeException(new ClusterStorageCapacityExceededException());
+ conf.setBoolean(KILL_LIMIT_EXCEED_CONF_NAME, false);
+
+ verifyReportError(exception, false);
+ }
+
+ @Test
+ public void testReportErrorWhenCapacityExceptionHappenAndFastFailEnabled()
+ throws IOException {
+ Exception exception =
+ new RuntimeException(new ClusterStorageCapacityExceededException());
+ conf.setBoolean(KILL_LIMIT_EXCEED_CONF_NAME, true);
+
+ verifyReportError(exception, true);
+ }
+
+ @Test
+ public void testReportErrorWhenCapacityExceptionHappenInThirdOfExceptionChain()
+ throws IOException {
+ Exception exception = new RuntimeException(new IllegalStateException(
+ new ClusterStorageCapacityExceededException()));
+ conf.setBoolean(KILL_LIMIT_EXCEED_CONF_NAME, true);
+
+ verifyReportError(exception, true);
+ }
+
+ private void verifyReportError(Exception exception, boolean fastFail)
+ throws IOException {
+ YarnChild.reportError(exception, task, umbilical);
+ verify(umbilical).fatalError(any(TaskAttemptID.class), anyString(),
+ eq(fastFail));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b6625a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 3592b3d..565c052 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -57,6 +57,9 @@ public interface MRJobConfig {
// negative values disable the limit
public static final long DEFAULT_JOB_SINGLE_DISK_LIMIT_BYTES = -1;
+ public static final String JOB_DFS_STORAGE_CAPACITY_KILL_LIMIT_EXCEED =
+ "mapreduce.job.dfs.storage.capacity.kill-limit-exceed";
+ public static final boolean DEFAULT_JOB_DFS_STORAGE_CAPACITY_KILL_LIMIT_EXCEED = false;
public static final String JOB_SINGLE_DISK_LIMIT_KILL_LIMIT_EXCEED =
"mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed";
// setting to false only logs the kill
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b6625a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index ccc9c3d..c993537 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -85,6 +85,15 @@
</property>
<property>
+ <name>mapreduce.job.dfs.storage.capacity.kill-limit-exceed</name>
+ <value>false</value>
+ <description>Whether to fast fail the task when exceeds allocated storage
+ capacity in the cluster filesystem(ClusterStorageCapacityExceededException
+ happens), for example, exceeds the dfs quota limitation. If true, the
+ task will fast fail. If false, the task will get retried.</description>
+</property>
+
+<property>
<name>mapreduce.job.maps</name>
<value>2</value>
<description>The default number of map tasks per job.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/29] hadoop git commit: HDFS-14047. [libhdfs++] Fix
hdfsGetLastExceptionRootCause bug in test_libhdfs_threaded.c (Contributed by
Antoli Shein via Daniel Templeton)
Posted by ae...@apache.org.
HDFS-14047. [libhdfs++] Fix hdfsGetLastExceptionRootCause bug in test_libhdfs_threaded.c
(Contributed by Antoli Shein via Daniel Templeton)
Change-Id: Ia5546fd5f8270f285c373b9023a9bd9835059845
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/349168c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/349168c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/349168c4
Branch: refs/heads/HDDS-4
Commit: 349168c4b34275d768e35e52bd3790a809f674a2
Parents: ab6aa4c
Author: Daniel Templeton <te...@apache.org>
Authored: Mon Nov 5 16:42:29 2018 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Nov 6 09:22:27 2018 -0800
----------------------------------------------------------------------
.../hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/349168c4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
index dbaf1f6..bcbb851 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
@@ -130,7 +130,7 @@ static char* getExceptionUtilString(JNIEnv *env, jthrowable exc, char *methodNam
jstring jStr = NULL;
char *excString = NULL;
jthr = invokeMethod(env, &jVal, STATIC, NULL,
- "org/apache/commons/lang/exception/ExceptionUtils",
+ "org/apache/commons/lang3/exception/ExceptionUtils",
methodName, "(Ljava/lang/Throwable;)Ljava/lang/String;", exc);
if (jthr) {
destroyLocalReference(env, jthr);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/29] hadoop git commit: HADOOP-15904. [JDK 11] Javadoc build
failed due to bad use of '>' in hadoop-hdfs.
Posted by ae...@apache.org.
HADOOP-15904. [JDK 11] Javadoc build failed due to bad use of '>' in hadoop-hdfs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6430c989
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6430c989
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6430c989
Branch: refs/heads/HDDS-4
Commit: 6430c9890aae46aeb2f41512ee442adfda5189dc
Parents: 5d6554c
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 6 18:42:38 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 6 18:42:38 2018 +0900
----------------------------------------------------------------------
.../hadoop/hdfs/server/diskbalancer/planner/package-info.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6430c989/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
index 0ad1cd9..4df9ba3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/package-info.java
@@ -29,7 +29,7 @@
*
* while(current.isBalancingNeeded(thresholdValue)) {
*
- * // Creates a plan , like move 20 GB data from v1 -> v2
+ * // Creates a plan , like move 20 GB data from v1 {@literal ->} v2
* Step step = planner.plan(current, thresholdValue);
*
* // we add that to our plan
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/29] hadoop git commit: HDFS-14053. Provide ability for NN to
re-replicate based on topology changes. Contributed by Hrishikesh Gadre.
Posted by ae...@apache.org.
HDFS-14053. Provide ability for NN to re-replicate based on topology changes. Contributed by Hrishikesh Gadre.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ffc9c50e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ffc9c50e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ffc9c50e
Branch: refs/heads/HDDS-4
Commit: ffc9c50e074aeca804674c6e1e6b0f1eb629e230
Parents: c7fcca0
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Nov 5 21:36:43 2018 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Nov 5 21:38:39 2018 -0800
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 38 +++++++++++
.../hdfs/server/namenode/NamenodeFsck.java | 33 +++++++++
.../org/apache/hadoop/hdfs/tools/DFSck.java | 10 ++-
.../src/site/markdown/HDFSCommands.md | 3 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 21 ++++++
.../TestBlocksWithNotEnoughRacks.java | 72 ++++++++++++++++++++
6 files changed, 173 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc9c50e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index a5fb0b1..36bbeb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3535,6 +3535,44 @@ public class BlockManager implements BlockStatsMXBean {
}
/**
+ * Schedule replication work for a specified list of mis-replicated
+ * blocks and return total number of blocks scheduled for replication.
+ *
+ * @param blocks A list of blocks for which replication work needs to
+ * be scheduled.
+ * @return Total number of blocks for which replication work is scheduled.
+ **/
+ public int processMisReplicatedBlocks(List<BlockInfo> blocks) {
+ int processed = 0;
+ Iterator<BlockInfo> iter = blocks.iterator();
+
+ try {
+ while (isPopulatingReplQueues() && namesystem.isRunning()
+ && !Thread.currentThread().isInterrupted()
+ && iter.hasNext()) {
+ int limit = processed + numBlocksPerIteration;
+ namesystem.writeLockInterruptibly();
+ try {
+ while (iter.hasNext() && processed < limit) {
+ BlockInfo blk = iter.next();
+ MisReplicationResult r = processMisReplicatedBlock(blk);
+ LOG.debug("BLOCK* processMisReplicatedBlocks: " +
+ "Re-scanned block {}, result is {}", blk, r);
+ }
+ } finally {
+ namesystem.writeUnlock();
+ }
+ }
+ } catch (InterruptedException ex) {
+ LOG.info("Caught InterruptedException while scheduling replication work" +
+ " for mis-replicated blocks");
+ Thread.currentThread().interrupt();
+ }
+
+ return processed;
+ }
+
+ /**
* Process a single possibly misreplicated block. This adds it to the
* appropriate queues if necessary, and returns a result code indicating
* what happened with it.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc9c50e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 56607f0..f54b407 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -25,6 +25,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.util.ArrayList;
+import java.util.LinkedList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
@@ -173,6 +174,14 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
*/
private boolean doDelete = false;
+ /**
+ * True if the user specified the -replicate option.
+ *
+ * When this option is in effect, we will initiate replication work to make
+ * mis-replicated blocks confirm the block placement policy.
+ */
+ private boolean doReplicate = false;
+
String path = "/";
private String blockIds = null;
@@ -249,6 +258,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
this.snapshottableDirs = new ArrayList<String>();
} else if (key.equals("blockId")) {
this.blockIds = pmap.get("blockId")[0];
+ } else if (key.equals("replicate")) {
+ this.doReplicate = true;
}
}
}
@@ -683,6 +694,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
StringBuilder report = new StringBuilder();
int blockNumber = 0;
final LocatedBlock lastBlock = blocks.getLastLocatedBlock();
+ List<BlockInfo> misReplicatedBlocks = new LinkedList<>();
for (LocatedBlock lBlk : blocks.getLocatedBlocks()) {
ExtendedBlock block = lBlk.getBlock();
if (!blocks.isLastBlockComplete() && lastBlock != null &&
@@ -791,6 +803,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
out.println(" Replica placement policy is violated for " +
block + ". " + blockPlacementStatus.getErrorDescription());
+ if (doReplicate) {
+ misReplicatedBlocks.add(storedBlock);
+ }
}
// count storage summary
@@ -888,6 +903,19 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.print(report + "\n");
}
}
+
+ if (doReplicate && !misReplicatedBlocks.isEmpty()) {
+ int processedBlocks = this.blockManager.processMisReplicatedBlocks(
+ misReplicatedBlocks);
+ if (processedBlocks < misReplicatedBlocks.size()) {
+ LOG.warn("Fsck: Block manager is able to process only " +
+ processedBlocks +
+ " mis-replicated blocks (Total count : " +
+ misReplicatedBlocks.size() +
+ " ) for path " + path);
+ }
+ res.numBlocksQueuedForReplication += processedBlocks;
+ }
}
private void countStorageTypeSummary(HdfsFileStatus file, LocatedBlock lBlk) {
@@ -1167,6 +1195,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
long totalSize = 0L;
long totalOpenFilesSize = 0L;
long totalReplicas = 0L;
+ long numBlocksQueuedForReplication = 0L;
/**
* DFS is considered healthy if there are no missing blocks.
@@ -1310,6 +1339,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.append("\n InMaintenanceReplicas:\t").append(
inMaintenanceReplicas);
}
+ res.append("\n Blocks queued for replication:\t").append(
+ numBlocksQueuedForReplication);
return res.toString();
}
}
@@ -1420,6 +1451,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.append("\n InMaintenanceReplicas:\t").append(
inMaintenanceReplicas);
}
+ res.append("\n Blocks queued for replication:\t").append(
+ numBlocksQueuedForReplication);
return res.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc9c50e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 2afc97c..403c0bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -84,7 +84,7 @@ public class DFSck extends Configured implements Tool {
"-upgradedomains]]]] "
+ "[-includeSnapshots] [-showprogress] "
+ "[-storagepolicies] [-maintenance] "
- + "[-blockId <blk_Id>]\n"
+ + "[-blockId <blk_Id>] [-replicate]\n"
+ "\t<path>\tstart checking from this path\n"
+ "\t-move\tmove corrupted files to /lost+found\n"
+ "\t-delete\tdelete corrupted files\n"
@@ -107,8 +107,10 @@ public class DFSck extends Configured implements Tool {
+ "\t-showprogress\tshow progress in output. Default is OFF (no progress)\n"
+ "\t-blockId\tprint out which file this blockId belongs to, locations"
+ " (nodes, racks) of this block, and other diagnostics info"
- + " (under replicated, corrupted or not, etc)\n\n"
- + "Please Note:\n"
+ + " (under replicated, corrupted or not, etc)\n"
+ + "\t-replicate initiate replication work to make mis-replicated\n"
+ + " blocks satisfy block placement policy\n\n"
+ + "Please Note:\n\n"
+ "\t1. By default fsck ignores files opened for write, "
+ "use -openforwrite to report such files. They are usually "
+ " tagged CORRUPT or HEALTHY depending on their block "
@@ -308,6 +310,8 @@ public class DFSck extends Configured implements Tool {
idx++;
}
url.append("&blockId=").append(URLEncoder.encode(sb.toString(), "UTF-8"));
+ } else if (args[idx].equals("-replicate")) {
+ url.append("&replicate=1");
} else if (!args[idx].startsWith("-")) {
if (null == dir) {
dir = args[idx];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc9c50e/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index c2a6ae1..012d4ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -86,7 +86,7 @@ Usage:
[-files [-blocks [-locations | -racks | -replicaDetails | -upgradedomains]]]
[-includeSnapshots] [-showprogress]
[-storagepolicies] [-maintenance]
- [-blockId <blk_Id>]
+ [-blockId <blk_Id>] [-replicate]
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -106,6 +106,7 @@ Usage:
| `-storagepolicies` | Print out storage policy summary for the blocks. |
| `-maintenance` | Print out maintenance state node details. |
| `-blockId` | Print out information about the block. |
+| `-replicate` | Initiate replication work to make mis-replicated blocks satisfy block placement policy. |
Runs the HDFS filesystem checking utility. See [fsck](./HdfsUserGuide.html#fsck) for more info.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc9c50e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 3e22b56..9465b31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -82,6 +82,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdfs.tools.DFSck;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -2531,4 +2532,24 @@ public class DFSTestUtil {
}
}
+ /**
+ * Run the fsck command using the specified params.
+ *
+ * @param conf HDFS configuration to use
+ * @param expectedErrCode The error code expected to be returned by
+ * the fsck command
+ * @param checkErrorCode Should the error code be checked
+ * @param path actual arguments to the fsck command
+ **/
+ public static String runFsck(Configuration conf, int expectedErrCode,
+ boolean checkErrorCode, String... path)
+ throws Exception {
+ ByteArrayOutputStream bStream = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(bStream, true);
+ int errCode = ToolRunner.run(new DFSck(conf, out), path);
+ if (checkErrorCode) {
+ assertEquals(expectedErrCode, errCode);
+ }
+ return bStream.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffc9c50e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
index 5e59443..85f8e54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
@@ -20,9 +20,14 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
+import java.util.concurrent.TimeoutException;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -144,6 +149,73 @@ public class TestBlocksWithNotEnoughRacks {
}
/*
+ * Initialize a cluster with datanodes on two different racks and shutdown
+ * all datanodes on one rack. Now create a file with a single block. Even
+ * though the block is sufficiently replicated, it violates the replica
+ * placement policy. Now restart the datanodes stopped earlier. Run the fsck
+ * command with -replicate option to schedule the replication of these
+ * mis-replicated blocks and verify if it indeed works as expected.
+ */
+ @Test
+ public void testMisReplicatedBlockUsesNewRack() throws Exception {
+ Configuration conf = getConf();
+ conf.setInt("dfs.namenode.heartbeat.recheck-interval", 500);
+
+ final short replicationFactor = 3;
+ final Path filePath = new Path("/testFile");
+ // All datanodes are on two different racks
+ String[] racks = new String[]{"/rack1", "/rack1", "/rack1", "/rack2"};
+
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(racks.length).racks(racks).build()) {
+ cluster.waitActive();
+
+ String poolId = cluster.getNamesystem().getBlockPoolId();
+ DatanodeRegistration reg = InternalDataNodeTestUtils.
+ getDNRegistrationForBP(cluster.getDataNodes().get(3), poolId);
+ // Shutdown datanode on rack2 and wait for it to be marked dead
+ cluster.stopDataNode(3);
+ DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(),
+ false, 20000);
+
+ // Create a file with one block with a replication factor of 3
+ final FileSystem fs = cluster.getFileSystem();
+ DFSTestUtil.createFile(fs, filePath, 1L, replicationFactor, 1L);
+ ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
+ DFSTestUtil.waitReplication(cluster.getFileSystem(), filePath,
+ replicationFactor);
+
+ // Add datanode on rack2 and wait for it be recognized as alive by NN
+ cluster.startDataNodes(conf, 1, true,
+ null, new String[]{"/rack2"});
+ cluster.waitActive();
+
+ try {
+ DFSTestUtil.waitForReplication(cluster, b, 2, replicationFactor, 0);
+ fail("NameNode should not have fixed the mis-replicated blocks" +
+ " automatically.");
+ } catch (TimeoutException e) {
+ //Expected.
+ }
+
+ String fsckOp = DFSTestUtil.runFsck(conf, 0, true, filePath.toString(),
+ "-replicate");
+ LOG.info("fsck response {}", fsckOp);
+ assertTrue(fsckOp.contains(
+ "/testFile: Replica placement policy is violated"));
+ assertTrue(fsckOp.contains(" Block should be additionally replicated" +
+ " on 1 more rack(s). Total number of racks in the cluster: 2"));
+
+ try {
+ DFSTestUtil.waitForReplication(cluster, b, 2, replicationFactor, 0);
+ } catch (TimeoutException e) {
+ fail("NameNode should have fixed the mis-replicated blocks as a" +
+ " result of fsck command.");
+ }
+ }
+ }
+
+ /*
* Creates a block with all datanodes on the same rack. Add additional
* datanodes on a different rack and increase the replication factor,
* making sure there are enough replicas across racks. If the previous
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/29] hadoop git commit: YARN-8880. Add configurations for
pluggable plugin framework. Contributed by Zhankun Tang.
Posted by ae...@apache.org.
YARN-8880. Add configurations for pluggable plugin framework. Contributed by Zhankun Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8c72d7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8c72d7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8c72d7b
Branch: refs/heads/HDDS-4
Commit: f8c72d7b3acca8285bbc3024f491c4586805be1e
Parents: c96cbe8
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Nov 8 12:23:00 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Nov 8 12:23:00 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 24 ++++-
.../src/main/resources/yarn-default.xml | 19 ++++
.../resourceplugin/ResourcePluginManager.java | 34 +++++-
.../TestResourcePluginManager.java | 107 +++++++++++++++++--
4 files changed, 170 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ce38d27..e88d594 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1606,6 +1606,28 @@ public class YarnConfiguration extends Configuration {
NM_PREFIX + "resource-plugins";
/**
+ * This setting controls if pluggable device plugin framework is enabled.
+ * */
+ @Private
+ public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED =
+ NM_PREFIX + "pluggable-device-framework.enabled";
+
+ /**
+ * The pluggable device plugin framework is disabled by default
+ * */
+ @Private
+ public static final boolean DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED =
+ false;
+
+ /**
+ * This setting contains vendor plugin class names for
+ * device plugin framework to load. Split by comma
+ * */
+ @Private
+ public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES =
+ NM_PREFIX + "pluggable-device-framework.device-classes";
+
+ /**
* Prefix for gpu configurations. Work in progress: This configuration
* parameter may be changed/removed in the future.
*/
@@ -1647,7 +1669,7 @@ public class YarnConfiguration extends Configuration {
NVIDIA_DOCKER_V1;
/**
- * This setting controls end point of nvidia-docker-v1 plugin
+ * This setting controls end point of nvidia-docker-v1 plugin.
*/
@Private
public static final String NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1360e73..f5493bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3772,6 +3772,25 @@
<property>
<description>
+ This setting controls if pluggable device framework is enabled.
+ Disabled by default
+ </description>
+ <name>yarn.nodemanager.pluggable-device-framework.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ Configure vendor device plugin class name here. Comma separated.
+ The class must be found in CLASSPATH. The pluggable device framework will
+ load these classes.
+ </description>
+ <name>yarn.nodemanager.pluggable-device-framework.device-classes</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified,
YARN NodeManager needs to run GPU discovery binary (now only support
nvidia-smi) to get GPU-related information.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index f28aad2..e930d96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
@@ -52,11 +53,10 @@ public class ResourcePluginManager {
public synchronized void initialize(Context context)
throws YarnException {
Configuration conf = context.getConf();
- String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
+ Map<String, ResourcePlugin> pluginMap = new HashMap<>();
+ String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
if (plugins != null) {
- Map<String, ResourcePlugin> pluginMap = new HashMap<>();
-
// Initialize each plugins
for (String resourceName : plugins) {
resourceName = resourceName.trim();
@@ -92,8 +92,34 @@ public class ResourcePluginManager {
plugin.initialize(context);
pluginMap.put(resourceName, plugin);
}
+ }
+ // Try to load pluggable device plugins
+ boolean puggableDeviceFrameworkEnabled = conf.getBoolean(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ YarnConfiguration.DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+
+ if (puggableDeviceFrameworkEnabled) {
+ initializePluggableDevicePlugins(context, conf, pluginMap);
+ } else {
+ LOG.info("The pluggable device framework is not enabled."
+ + " If you want, please set true to {}",
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+ }
+ configuredPlugins = Collections.unmodifiableMap(pluginMap);
+ }
- configuredPlugins = Collections.unmodifiableMap(pluginMap);
+ public void initializePluggableDevicePlugins(Context context,
+ Configuration configuration,
+ Map<String, ResourcePlugin> pluginMap)
+ throws YarnRuntimeException {
+ LOG.info("The pluggable device framework enabled," +
+ "trying to load the vendor plugins");
+
+ String[] pluginClassNames = configuration.getStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
+ if (null == pluginClassNames) {
+ throw new YarnRuntimeException("Null value found in configuration: "
+ + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
index 6ed7c56..eb8bf54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -41,12 +42,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import java.util.HashMap;
@@ -58,10 +57,18 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
public class TestResourcePluginManager extends NodeManagerTestBase {
private NodeManager nm;
+ private YarnConfiguration conf;
+
+ @Before
+ public void setup() throws Exception {
+ this.conf = createNMConfig();
+ }
+
ResourcePluginManager stubResourcePluginmanager() {
// Stub ResourcePluginManager
final ResourcePluginManager rpm = mock(ResourcePluginManager.class);
@@ -183,7 +190,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
final ResourcePluginManager rpm = stubResourcePluginmanager();
nm = new MyMockNM(rpm);
- YarnConfiguration conf = createNMConfig();
nm.init(conf);
verify(rpm, times(1)).initialize(
any(Context.class));
@@ -198,7 +204,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
nm = new MyMockNM(rpm);
- YarnConfiguration conf = createNMConfig();
nm.init(conf);
nm.start();
@@ -238,15 +243,14 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
}
@Override
- protected ContainerExecutor createContainerExecutor(Configuration conf) {
+ protected ContainerExecutor createContainerExecutor(
+ Configuration configuration) {
((NMContext)this.getNMContext()).setResourcePluginManager(rpm);
- lce.setConf(conf);
+ lce.setConf(configuration);
return lce;
}
};
- YarnConfiguration conf = createNMConfig();
-
nm.init(conf);
nm.start();
@@ -264,4 +268,89 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
}
Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded);
}
+
+ // Disabled pluggable framework in configuration.
+ // We use spy object of real rpm to verify "initializePluggableDevicePlugins"
+ // because use mock rpm will not working
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkDisabled()
+ throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ false);
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(0)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // No related configuration set.
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkDisabled2()
+ throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(0)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // Enable framework and configure pluggable device classes
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkEnabled()
+ throws Exception {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+ conf.setStrings(
+ YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+ "com.cmp1.hdw1plugin");
+ nm.init(conf);
+ nm.start();
+ verify(rpmSpy, times(1)).initialize(
+ any(Context.class));
+ verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ }
+
+ // Enable pluggable framework, but leave device classes un-configured
+ // initializePluggableDevicePlugins invoked but it should throw an exception
+ @Test(timeout = 30000)
+ public void testInitializationWithPluggableDeviceFrameworkEnabled2() {
+ ResourcePluginManager rpm = new ResourcePluginManager();
+
+ ResourcePluginManager rpmSpy = spy(rpm);
+ nm = new MyMockNM(rpmSpy);
+ Boolean fail = false;
+ try {
+ conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+ true);
+
+ nm.init(conf);
+ nm.start();
+ } catch (YarnRuntimeException e) {
+ fail = true;
+ } catch (Exception e) {
+
+ }
+ verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+ any(Context.class), any(Configuration.class), any(Map.class));
+ Assert.assertTrue(fail);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/29] hadoop git commit: YARN-8866. Fix a parsing error for
crossdomain.xml.
Posted by ae...@apache.org.
YARN-8866. Fix a parsing error for crossdomain.xml.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8dc1f6db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8dc1f6db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8dc1f6db
Branch: refs/heads/HDDS-4
Commit: 8dc1f6dbf712a65390a9a6859f62fec0481af31b
Parents: e6444f1
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Wed Nov 7 18:26:07 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Wed Nov 7 18:26:07 2018 +0900
----------------------------------------------------------------------
.../hadoop-yarn-ui/src/main/webapp/public/crossdomain.xml | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dc1f6db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/public/crossdomain.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/public/crossdomain.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/public/crossdomain.xml
index 43a2ea6..a9597e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/public/crossdomain.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/public/crossdomain.xml
@@ -18,7 +18,12 @@
* limitations under the License.
-->
-<!DOCTYPE cross-domain-policy SYSTEM "http://www.adobe.com/xml/dtds/cross-domain-policy.dtd">
+<!DOCTYPE cross-domain-policy [
+ <!ELEMENT cross-domain-policy (site-control)*>
+ <!ELEMENT site-control (#PCDATA)>
+ <!ATTLIST site-control
+ permitted-cross-domain-policies CDATA #REQUIRED>
+ ]>
<cross-domain-policy>
<!-- Read this: www.adobe.com/devnet/articles/crossdomain_policy_file_spec.html -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/29] hadoop git commit: HADOOP-15902. [JDK 11] Specify the HTML
version of Javadoc to 4.01. Contributed by Takanobu Asanuma.
Posted by ae...@apache.org.
HADOOP-15902. [JDK 11] Specify the HTML version of Javadoc to 4.01. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78e43bcc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78e43bcc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78e43bcc
Branch: refs/heads/HDDS-4
Commit: 78e43bccf655153d5fc8fc891d59c536354d8031
Parents: 6430c98
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 6 20:44:05 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 6 20:44:05 2018 +0900
----------------------------------------------------------------------
hadoop-project/pom.xml | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78e43bcc/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 937f21f..5d38167 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -2002,6 +2002,26 @@
</dependencies>
</dependencyManagement>
</profile>
+ <profile>
+ <!-- TODO: remove this profile to generate html5 docs when we stop supporting JDK8 -->
+ <id>javadoc-html-version</id>
+ <activation>
+ <jdk>[11,)</jdk>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <additionalOptions>
+ <additionalOption>-html4</additionalOption>
+ </additionalOptions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<repositories>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/29] hadoop git commit: Merge branch 'trunk' into HDDS-4
Posted by ae...@apache.org.
Merge branch 'trunk' into HDDS-4
Conflicts:
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/629347be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/629347be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/629347be
Branch: refs/heads/HDDS-4
Commit: 629347beadf0bc69c32c61e895ad03cb082df81a
Parents: 10d3954 724c150
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Nov 8 10:10:50 2018 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Nov 8 10:10:50 2018 -0800
----------------------------------------------------------------------
BUILDING.txt | 30 +-
...ClusterStorageCapacityExceededException.java | 51 +++
.../AbstractDelegationTokenSecretManager.java | 12 +-
hadoop-hdds/common/pom.xml | 6 +
.../hdds/scm/container/ContainerInfo.java | 8 +
.../statemachine/DatanodeStateMachine.java | 17 +-
.../common/statemachine/StateContext.java | 13 +-
.../CloseContainerCommandHandler.java | 73 ++---
.../states/endpoint/HeartbeatEndpointTask.java | 12 +-
.../StorageContainerDatanodeProtocol.proto | 2 +-
.../org/apache/hadoop/hdds/scm/ScmUtils.java | 2 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 2 +-
.../hdds/scm/chillmode/ChillModeExitRule.java | 32 ++
.../hdds/scm/chillmode/ChillModePrecheck.java | 68 ++++
.../scm/chillmode/ChillModeRestrictedOps.java | 41 +++
.../scm/chillmode/ContainerChillModeRule.java | 112 +++++++
.../scm/chillmode/DataNodeChillModeRule.java | 83 +++++
.../hadoop/hdds/scm/chillmode/Precheck.java | 29 ++
.../hdds/scm/chillmode/SCMChillModeManager.java | 153 +++++++++
.../hadoop/hdds/scm/chillmode/package-info.java | 18 ++
.../scm/command/CommandStatusReportHandler.java | 4 -
.../container/CloseContainerEventHandler.java | 27 --
.../scm/container/CloseContainerWatcher.java | 101 ------
.../hdds/scm/container/ContainerManager.java | 11 -
.../hdds/scm/container/ContainerReplica.java | 30 +-
.../scm/container/ContainerReportHandler.java | 202 ++++++++----
.../scm/container/ContainerStateManager.java | 4 +-
.../IncrementalContainerReportHandler.java | 98 ++++++
.../hdds/scm/container/SCMContainerManager.java | 107 -------
.../scm/container/states/ContainerStateMap.java | 7 -
.../hadoop/hdds/scm/events/SCMEvents.java | 34 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 37 ++-
.../hadoop/hdds/scm/node/NewNodeHandler.java | 16 +-
.../hadoop/hdds/scm/node/NodeManager.java | 50 +--
.../hadoop/hdds/scm/node/NodeReportHandler.java | 2 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 69 +---
.../hadoop/hdds/scm/node/SCMNodeManager.java | 66 +---
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 2 +-
.../hdds/scm/node/states/NodeStateMap.java | 67 ++--
.../hdds/scm/pipeline/PipelineManager.java | 3 +
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../hdds/scm/pipeline/PipelineStateMap.java | 22 ++
.../hdds/scm/pipeline/SCMPipelineManager.java | 11 +
.../hdds/scm/server/ChillModePrecheck.java | 69 ----
.../apache/hadoop/hdds/scm/server/Precheck.java | 29 --
.../hdds/scm/server/SCMChillModeManager.java | 319 -------------------
.../scm/server/SCMClientProtocolServer.java | 1 +
.../server/SCMDatanodeHeartbeatDispatcher.java | 17 +-
.../scm/server/SCMDatanodeProtocolServer.java | 6 +-
.../scm/server/StorageContainerManager.java | 22 +-
.../scm/chillmode/TestSCMChillModeManager.java | 215 +++++++++++++
.../command/TestCommandStatusReportHandler.java | 14 -
.../hdds/scm/container/MockNodeManager.java | 66 ++--
.../container/TestContainerReportHandler.java | 15 +-
.../container/TestContainerStateManager.java | 3 +
.../scm/container/TestSCMContainerManager.java | 107 +------
.../replication/TestReplicationManager.java | 12 +-
.../hdds/scm/node/TestDeadNodeHandler.java | 24 +-
.../scm/server/TestSCMChillModeManager.java | 215 -------------
.../container/TestCloseContainerWatcher.java | 289 -----------------
.../ozone/container/common/TestEndPoint.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 55 +---
.../org/apache/hadoop/hdfs/DataStreamer.java | 1 +
.../hdfs/protocol/QuotaExceededException.java | 5 +-
.../src/main/native/libhdfs/exception.c | 2 +-
.../federation/router/RouterHttpServer.java | 4 +-
.../server/blockmanagement/BlockManager.java | 38 +++
.../diskbalancer/planner/package-info.java | 2 +-
.../server/namenode/NameNodeHttpServer.java | 13 +-
.../hdfs/server/namenode/NamenodeFsck.java | 33 ++
.../org/apache/hadoop/hdfs/tools/DFSck.java | 10 +-
.../hadoop/hdfs/util/PersistentLongFile.java | 2 +
.../src/site/markdown/HDFSCommands.md | 3 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 21 ++
.../TestClientProtocolForPipelineRecovery.java | 2 +
.../TestBlocksWithNotEnoughRacks.java | 72 +++++
.../hdfs/server/namenode/TestSaveNamespace.java | 56 ++++
.../org/apache/hadoop/mapred/YarnChild.java | 28 +-
.../org/apache/hadoop/mapred/TestYarnChild.java | 118 +++++++
.../apache/hadoop/mapreduce/MRJobConfig.java | 3 +
.../src/main/resources/mapred-default.xml | 9 +
.../hs/TestJHSDelegationTokenSecretManager.java | 78 ++++-
.../apache/hadoop/mapred/ShuffleHandler.java | 4 +-
.../apache/hadoop/ozone/client/OzoneKey.java | 16 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 9 +-
.../hadoop/ozone/client/rest/RestClient.java | 6 +-
.../hadoop/ozone/client/rpc/RpcClient.java | 14 +-
.../ozone/client/rest/response/KeyInfo.java | 20 ++
.../hadoop/ozone/web/response/KeyInfo.java | 21 ++
.../main/smoketest/s3/objectmultidelete.robot | 4 +
.../TestContainerStateManagerIntegration.java | 10 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 4 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 4 +-
.../scm/pipeline/TestPipelineStateManager.java | 61 +++-
.../commandhandler/TestBlockDeletion.java | 6 +-
.../hadoop/ozone/om/TestScmChillMode.java | 2 +-
hadoop-ozone/s3gateway/pom.xml | 4 +
.../ozone/s3/endpoint/BucketEndpoint.java | 9 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
.../ozone/s3/TestSignedChunksInputStream.java | 2 +-
hadoop-project/pom.xml | 20 ++
.../fs/azurebfs/AzureBlobFileSystemStore.java | 20 +-
.../fs/azurebfs/services/AbfsAclHelper.java | 89 +++++-
.../azurebfs/ITestAzureBlobFilesystemAcl.java | 147 ++++++++-
.../hadoop/yarn/api/ApplicationConstants.java | 22 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 24 +-
.../api/records/ComponentContainers.java | 3 +-
.../src/main/resources/yarn-default.xml | 19 ++
.../resourceplugin/ResourcePluginManager.java | 34 +-
.../TestResourcePluginManager.java | 107 ++++++-
.../nodelabels/NodeAttributesManagerImpl.java | 6 +-
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 96 ++++--
.../scheduler/fair/FSPreemptionThread.java | 2 +-
.../TestCapacitySchedulerAsyncScheduling.java | 83 +++++
.../fair/TestContinuousScheduling.java | 3 +-
.../scheduler/fair/TestFairScheduler.java | 3 +-
.../security/TestRMDelegationTokens.java | 63 ++++
.../src/main/webapp/public/crossdomain.xml | 7 +-
119 files changed, 2713 insertions(+), 1958 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/629347be/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/629347be/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/629347be/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/629347be/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/629347be/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/29] hadoop git commit: HADOOP-15846. ABFS: fix mask related bugs
in setAcl, modifyAclEntries and removeAclEntries.
Posted by ae...@apache.org.
HADOOP-15846. ABFS: fix mask related bugs in setAcl, modifyAclEntries and removeAclEntries.
Contributed by Junhua Gu.
(cherry picked from commit 66715005f9e8f4f25faa352a06d142b75a029f0e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/724c1500
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/724c1500
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/724c1500
Branch: refs/heads/HDDS-4
Commit: 724c15007beb9ca36f2f37c829f1daa366d95bbf
Parents: 9317a61
Author: Junhua Gu <Ju...@microsoft.com>
Authored: Thu Nov 8 17:21:40 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 8 17:21:40 2018 +0000
----------------------------------------------------------------------
.../fs/azurebfs/AzureBlobFileSystemStore.java | 20 +--
.../fs/azurebfs/services/AbfsAclHelper.java | 89 +++++++++--
.../azurebfs/ITestAzureBlobFilesystemAcl.java | 147 ++++++++++++++++++-
3 files changed, 225 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/724c1500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 1ac1761..bfdbba8 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -626,17 +626,7 @@ public class AzureBlobFileSystemStore {
final Map<String, String> aclEntries = AbfsAclHelper.deserializeAclSpec(op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_ACL));
- for (Map.Entry<String, String> modifyAclEntry : modifyAclEntries.entrySet()) {
- aclEntries.put(modifyAclEntry.getKey(), modifyAclEntry.getValue());
- }
-
- if (!modifyAclEntries.containsKey(AbfsHttpConstants.ACCESS_MASK)) {
- aclEntries.remove(AbfsHttpConstants.ACCESS_MASK);
- }
-
- if (!modifyAclEntries.containsKey(AbfsHttpConstants.DEFAULT_MASK)) {
- aclEntries.remove(AbfsHttpConstants.DEFAULT_MASK);
- }
+ AbfsAclHelper.modifyAclEntriesInternal(aclEntries, modifyAclEntries);
client.setAcl(AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path, true),
AbfsAclHelper.serializeAclSpec(aclEntries), eTag);
@@ -736,12 +726,8 @@ public class AzureBlobFileSystemStore {
final String eTag = op.getResult().getResponseHeader(HttpHeaderConfigurations.ETAG);
final Map<String, String> getAclEntries = AbfsAclHelper.deserializeAclSpec(op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_ACL));
- for (Map.Entry<String, String> ace : getAclEntries.entrySet()) {
- if (ace.getKey().startsWith("default:") && (ace.getKey() != AbfsHttpConstants.DEFAULT_MASK)
- && !aclEntries.containsKey(ace.getKey())) {
- aclEntries.put(ace.getKey(), ace.getValue());
- }
- }
+
+ AbfsAclHelper.setAclEntriesInternal(aclEntries, getAclEntries);
client.setAcl(AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path, true),
AbfsAclHelper.serializeAclSpec(aclEntries), eTag);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/724c1500/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAclHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAclHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAclHelper.java
index c28da2c..34959a6 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAclHelper.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAclHelper.java
@@ -44,12 +44,17 @@ public final class AbfsAclHelper {
// not called
}
- public static Map<String, String> deserializeAclSpec(final String aclSpecString) {
+ public static Map<String, String> deserializeAclSpec(final String aclSpecString) throws AzureBlobFileSystemException {
final Map<String, String> aclEntries = new HashMap<>();
- final String[] aclArray = aclSpecString.split(AbfsHttpConstants.COMMA);
- for (String acl : aclArray) {
- int idx = acl.lastIndexOf(AbfsHttpConstants.COLON);
- aclEntries.put(acl.substring(0, idx), acl.substring(idx + 1));
+ final String[] aceArray = aclSpecString.split(AbfsHttpConstants.COMMA);
+ for (String ace : aceArray) {
+ int idx = ace.lastIndexOf(AbfsHttpConstants.COLON);
+ final String key = ace.substring(0, idx);
+ final String val = ace.substring(idx + 1);
+ if (aclEntries.containsKey(key)) {
+ throw new InvalidAclOperationException("Duplicate acl entries are not allowed.");
+ }
+ aclEntries.put(key, val);
}
return aclEntries;
}
@@ -104,12 +109,22 @@ public final class AbfsAclHelper {
}
}
}
+
+ if (removeIndicationSet.contains(AbfsHttpConstants.ACCESS_MASK) && containsNamedAce(aclEntries, false)) {
+ throw new InvalidAclOperationException("Access mask is required when a named access acl is present.");
+ }
+
if (accessAclTouched) {
if (removeIndicationSet.contains(AbfsHttpConstants.ACCESS_MASK)) {
aclEntries.remove(AbfsHttpConstants.ACCESS_MASK);
}
recalculateMask(aclEntries, false);
}
+
+ if (removeIndicationSet.contains(AbfsHttpConstants.DEFAULT_MASK) && containsNamedAce(aclEntries, true)) {
+ throw new InvalidAclOperationException("Default mask is required when a named default acl is present.");
+ }
+
if (defaultAclTouched) {
if (removeIndicationSet.contains(AbfsHttpConstants.DEFAULT_MASK)) {
aclEntries.remove(AbfsHttpConstants.DEFAULT_MASK);
@@ -127,6 +142,50 @@ public final class AbfsAclHelper {
}
}
+ public static void modifyAclEntriesInternal(Map<String, String> aclEntries, Map<String, String> toModifyEntries)
+ throws AzureBlobFileSystemException {
+ boolean namedAccessAclTouched = false;
+ boolean namedDefaultAclTouched = false;
+
+ for (Map.Entry<String, String> toModifyEntry : toModifyEntries.entrySet()) {
+ aclEntries.put(toModifyEntry.getKey(), toModifyEntry.getValue());
+ if (isNamedAce(toModifyEntry.getKey())) {
+ if (isDefaultAce(toModifyEntry.getKey())) {
+ namedDefaultAclTouched = true;
+ } else {
+ namedAccessAclTouched = true;
+ }
+ }
+ }
+
+ if (!toModifyEntries.containsKey(AbfsHttpConstants.ACCESS_MASK) && namedAccessAclTouched) {
+ aclEntries.remove(AbfsHttpConstants.ACCESS_MASK);
+ }
+
+ if (!toModifyEntries.containsKey(AbfsHttpConstants.DEFAULT_MASK) && namedDefaultAclTouched) {
+ aclEntries.remove(AbfsHttpConstants.DEFAULT_MASK);
+ }
+ }
+
+ public static void setAclEntriesInternal(Map<String, String> aclEntries, Map<String, String> getAclEntries)
+ throws AzureBlobFileSystemException {
+ boolean defaultAclTouched = false;
+
+ for (String entryKey : aclEntries.keySet()) {
+ if (isDefaultAce(entryKey)) {
+ defaultAclTouched = true;
+ break;
+ }
+ }
+
+ for (Map.Entry<String, String> ace : getAclEntries.entrySet()) {
+ if (AbfsAclHelper.isDefaultAce(ace.getKey()) && (ace.getKey() != AbfsHttpConstants.DEFAULT_MASK || !defaultAclTouched)
+ && !aclEntries.containsKey(ace.getKey())) {
+ aclEntries.put(ace.getKey(), ace.getValue());
+ }
+ }
+ }
+
private static boolean removeNamedAceAndUpdateSet(String entry, boolean isDefaultAcl, Set<String> removeIndicationSet,
Map<String, String> aclEntries)
throws AzureBlobFileSystemException {
@@ -136,8 +195,7 @@ public final class AbfsAclHelper {
: entryParts[startIndex] + AbfsHttpConstants.COLON;
if ((entry.equals(AbfsHttpConstants.ACCESS_USER) || entry.equals(AbfsHttpConstants.ACCESS_GROUP)
- || entry.equals(AbfsHttpConstants.ACCESS_OTHER))
- && !isNamedAce(entry)) {
+ || entry.equals(AbfsHttpConstants.ACCESS_OTHER))) {
throw new InvalidAclOperationException("Cannot remove user, group or other entry from access ACL.");
}
@@ -154,7 +212,7 @@ public final class AbfsAclHelper {
}
private static void recalculateMask(Map<String, String> aclEntries, boolean isDefaultMask) {
- FsAction umask = FsAction.NONE;
+ FsAction mask = FsAction.NONE;
if (!isExtendAcl(aclEntries, isDefaultMask)) {
return;
}
@@ -163,17 +221,17 @@ public final class AbfsAclHelper {
if (isDefaultMask) {
if ((isDefaultAce(aclEntry.getKey()) && isNamedAce(aclEntry.getKey()))
|| aclEntry.getKey().equals(AbfsHttpConstants.DEFAULT_GROUP)) {
- umask = umask.or(FsAction.getFsAction(aclEntry.getValue()));
+ mask = mask.or(FsAction.getFsAction(aclEntry.getValue()));
}
} else {
if ((!isDefaultAce(aclEntry.getKey()) && isNamedAce(aclEntry.getKey()))
|| aclEntry.getKey().equals(AbfsHttpConstants.ACCESS_GROUP)) {
- umask = umask.or(FsAction.getFsAction(aclEntry.getValue()));
+ mask = mask.or(FsAction.getFsAction(aclEntry.getValue()));
}
}
}
- aclEntries.put(isDefaultMask ? AbfsHttpConstants.DEFAULT_MASK : AbfsHttpConstants.ACCESS_MASK, umask.SYMBOL);
+ aclEntries.put(isDefaultMask ? AbfsHttpConstants.DEFAULT_MASK : AbfsHttpConstants.ACCESS_MASK, mask.SYMBOL);
}
private static boolean isExtendAcl(Map<String, String> aclEntries, boolean checkDefault) {
@@ -192,6 +250,15 @@ public final class AbfsAclHelper {
return false;
}
+ private static boolean containsNamedAce(Map<String, String> aclEntries, boolean checkDefault) {
+ for (String entryKey : aclEntries.keySet()) {
+ if (isNamedAce(entryKey) && (checkDefault == isDefaultAce(entryKey))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private static boolean isDefaultAce(String entry) {
return entry.startsWith(AbfsHttpConstants.DEFAULT_SCOPE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/724c1500/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
index 7377132..05f9856 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
@@ -30,6 +30,7 @@ import org.junit.Test;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -54,6 +55,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
private static final FsAction ALL = FsAction.ALL;
private static final FsAction NONE = FsAction.NONE;
private static final FsAction READ = FsAction.READ;
+ private static final FsAction EXECUTE = FsAction.EXECUTE;
private static final FsAction READ_EXECUTE = FsAction.READ_EXECUTE;
private static final FsAction READ_WRITE = FsAction.READ_WRITE;
@@ -65,6 +67,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
private static final short RWX_RX = 0750;
private static final short RWX_RX_RX = 0755;
private static final short RW_R = 0640;
+ private static final short RW_X = 0610;
private static final short RW_RW = 0660;
private static final short RW_RWX = 0670;
private static final short RW_R_R = 0644;
@@ -271,6 +274,67 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
}
@Test
+ public void testModifyAclEntriesWithDefaultMask() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, MASK, EXECUTE));
+ fs.setAcl(path, aclSpec);
+
+ List<AclEntry> modifyAclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, READ_WRITE));
+ fs.modifyAclEntries(path, modifyAclSpec);
+
+ AclStatus s = fs.getAclStatus(path);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[] {
+ aclEntry(DEFAULT, USER, READ_WRITE),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, MASK, EXECUTE),
+ aclEntry(DEFAULT, OTHER, NONE)}, returned);
+ assertPermission(fs, (short) RWX_RX);
+ }
+
+ @Test
+ public void testModifyAclEntriesWithAccessMask() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(ACCESS, MASK, EXECUTE));
+ fs.setAcl(path, aclSpec);
+
+ List<AclEntry> modifyAclSpec = Lists.newArrayList(
+ aclEntry(ACCESS, USER, READ_WRITE));
+ fs.modifyAclEntries(path, modifyAclSpec);
+
+ AclStatus s = fs.getAclStatus(path);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[] {
+ aclEntry(ACCESS, GROUP, READ_EXECUTE)}, returned);
+ assertPermission(fs, (short) RW_X);
+ }
+
+ @Test(expected=PathIOException.class)
+ public void testModifyAclEntriesWithDuplicateEntries() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(ACCESS, MASK, EXECUTE));
+ fs.setAcl(path, aclSpec);
+
+ List<AclEntry> modifyAclSpec = Lists.newArrayList(
+ aclEntry(ACCESS, USER, READ_WRITE),
+ aclEntry(ACCESS, USER, READ_WRITE));
+ fs.modifyAclEntries(path, modifyAclSpec);
+ }
+
+ @Test
public void testRemoveAclEntries() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
assumeTrue(fs.getIsNamespaceEnabled());
@@ -440,6 +504,50 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
fs.removeAclEntries(path, aclSpec);
}
+ @Test(expected=PathIOException.class)
+ public void testRemoveAclEntriesAccessMask() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(ACCESS, MASK, EXECUTE),
+ aclEntry(ACCESS, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ fs.removeAclEntries(path, Lists.newArrayList(aclEntry(ACCESS, MASK, NONE)));
+ }
+
+ @Test(expected=PathIOException.class)
+ public void testRemoveAclEntriesDefaultMask() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, MASK, EXECUTE),
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ fs.removeAclEntries(path, Lists.newArrayList(aclEntry(DEFAULT, MASK, NONE)));
+ }
+
+ @Test(expected=PathIOException.class)
+ public void testRemoveAclEntriesWithDuplicateEntries() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, MASK, EXECUTE));
+ fs.setAcl(path, aclSpec);
+
+ List<AclEntry> removeAclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, READ_WRITE),
+ aclEntry(DEFAULT, USER, READ_WRITE));
+ fs.removeAclEntries(path, removeAclSpec);
+ }
+
@Test
public void testRemoveDefaultAcl() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
@@ -814,6 +922,42 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
}
@Test
+ public void testSetAclDoesNotChangeDefaultMask() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, MASK, EXECUTE));
+ fs.setAcl(path, aclSpec);
+ // only change access acl, and default mask should not change.
+ List<AclEntry> aclSpec2 = Lists.newArrayList(
+ aclEntry(ACCESS, OTHER, READ_EXECUTE));
+ fs.setAcl(path, aclSpec2);
+ // get acl status and check result.
+ AclStatus s = fs.getAclStatus(path);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[] {
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, GROUP, READ_EXECUTE),
+ aclEntry(DEFAULT, MASK, EXECUTE),
+ aclEntry(DEFAULT, OTHER, NONE) }, returned);
+ assertPermission(fs, (short) RWX_RX_RX);
+ }
+
+ @Test(expected=PathIOException.class)
+ public void testSetAclWithDuplicateEntries() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ assumeTrue(fs.getIsNamespaceEnabled());
+ path = new Path(testRoot, UUID.randomUUID().toString());
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(ACCESS, MASK, EXECUTE),
+ aclEntry(ACCESS, MASK, EXECUTE));
+ fs.setAcl(path, aclSpec);
+ }
+
+ @Test
public void testSetPermission() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
assumeTrue(fs.getIsNamespaceEnabled());
@@ -924,7 +1068,6 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
}
@Test
- @Ignore // wait investigation in service
public void testDefaultMinimalAclNewFile() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
assumeTrue(fs.getIsNamespaceEnabled());
@@ -970,7 +1113,6 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
}
@Test
- @Ignore // wait umask fix to be deployed
public void testOnlyAccessAclNewDir() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
assumeTrue(fs.getIsNamespaceEnabled());
@@ -988,7 +1130,6 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
}
@Test
- @Ignore // wait investigation in service
public void testDefaultMinimalAclNewDir() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
assumeTrue(fs.getIsNamespaceEnabled());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/29] hadoop git commit: HDFS-14051. Refactor
NameNodeHttpServer#initWebHdfs to specify local keytab. Contributed by CR
Hota.
Posted by ae...@apache.org.
HDFS-14051. Refactor NameNodeHttpServer#initWebHdfs to specify local keytab. Contributed by CR Hota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0ce0729
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0ce0729
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0ce0729
Branch: refs/heads/HDDS-4
Commit: f0ce072934515d39e9bf61744058ecad3393291e
Parents: 3d5cc11
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Nov 6 10:05:58 2018 -0800
Committer: Inigo Goiri <in...@apache.org>
Committed: Tue Nov 6 10:05:58 2018 -0800
----------------------------------------------------------------------
.../server/federation/router/RouterHttpServer.java | 4 ++--
.../hdfs/server/namenode/NameNodeHttpServer.java | 13 +++++++------
2 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0ce0729/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
index 0cf884f..d223e2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHttpServer.java
@@ -89,8 +89,8 @@ public class RouterHttpServer extends AbstractService {
this.httpServer = builder.build();
- NameNodeHttpServer.initWebHdfs(conf, httpAddress.getHostName(), httpServer,
- RouterWebHdfsMethods.class.getPackage().getName());
+ NameNodeHttpServer.initWebHdfs(conf, httpAddress.getHostName(), null,
+ httpServer, RouterWebHdfsMethods.class.getPackage().getName());
this.httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, this.router);
this.httpServer.setAttribute(JspHelper.CURRENT_CONF, this.conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0ce0729/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index ae9c7fe..1bc43b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -77,6 +77,7 @@ public class NameNodeHttpServer {
}
public static void initWebHdfs(Configuration conf, String hostname,
+ String httpKeytab,
HttpServer2 httpServer2, String jerseyResourcePackage)
throws IOException {
// set user pattern based on configuration file
@@ -94,7 +95,8 @@ public class NameNodeHttpServer {
final String name = className;
final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
- Map<String, String> params = getAuthFilterParams(conf, hostname);
+ Map<String, String> params = getAuthFilterParams(conf, hostname,
+ httpKeytab);
HttpServer2.defineFilter(httpServer2.getWebAppContext(), name, className,
params, new String[] { pathSpec });
HttpServer2.LOG.info("Added filter '" + name + "' (class=" + className
@@ -167,8 +169,9 @@ public class NameNodeHttpServer {
httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY,
datanodeSslPort.getPort());
}
-
- initWebHdfs(conf, bindAddress.getHostName(), httpServer,
+ String httpKeytab = conf.get(DFSUtil.getSpnegoKeytabKey(conf,
+ DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
+ initWebHdfs(conf, bindAddress.getHostName(), httpKeytab, httpServer,
NamenodeWebHdfsMethods.class.getPackage().getName());
httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
@@ -191,7 +194,7 @@ public class NameNodeHttpServer {
}
private static Map<String, String> getAuthFilterParams(Configuration conf,
- String hostname) throws IOException {
+ String hostname, String httpKeytab) throws IOException {
Map<String, String> params = new HashMap<String, String>();
// Select configs beginning with 'dfs.web.authentication.'
Iterator<Map.Entry<String, String>> iterator = conf.iterator();
@@ -214,8 +217,6 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY +
"' is not set.");
}
- String httpKeytab = conf.get(DFSUtil.getSpnegoKeytabKey(conf,
- DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
if (httpKeytab != null && !httpKeytab.isEmpty()) {
params.put(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/29] hadoop git commit: YARN-8969.
AbstractYarnScheduler#getNodeTracker should return generic type to avoid type
casting. Contributed by Wanqiang Ji.
Posted by ae...@apache.org.
YARN-8969. AbstractYarnScheduler#getNodeTracker should return generic type to avoid type casting. Contributed by Wanqiang Ji.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7fcca0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7fcca0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7fcca0d
Branch: refs/heads/HDDS-4
Commit: c7fcca0d7ec9e31d43ef3040ecd576ec808f1f8b
Parents: f329650
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Nov 6 13:14:57 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Nov 6 13:14:57 2018 +0800
----------------------------------------------------------------------
.../server/resourcemanager/scheduler/AbstractYarnScheduler.java | 2 +-
.../server/resourcemanager/scheduler/fair/FSPreemptionThread.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7fcca0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 0acfca7..7acedf2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -237,7 +237,7 @@ public abstract class AbstractYarnScheduler
}
@VisibleForTesting
- public ClusterNodeTracker getNodeTracker() {
+ public ClusterNodeTracker<N> getNodeTracker() {
return nodeTracker;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7fcca0d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index c32565f..6ed90f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -229,7 +229,7 @@ class FSPreemptionThread extends Thread {
private void trackPreemptionsAgainstNode(List<RMContainer> containers,
FSAppAttempt app) {
- FSSchedulerNode node = (FSSchedulerNode) scheduler.getNodeTracker()
+ FSSchedulerNode node = scheduler.getNodeTracker()
.getNode(containers.get(0).getNodeId());
node.addContainersForPreemption(containers, app);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/29] hadoop git commit: MAPREDUCE-7156. NullPointerException when
reaching max shuffle connections. Contributed by Peter Bacsko
Posted by ae...@apache.org.
MAPREDUCE-7156. NullPointerException when reaching max shuffle connections. Contributed by Peter Bacsko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba1f9d66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba1f9d66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba1f9d66
Branch: refs/heads/HDDS-4
Commit: ba1f9d66d94ed0b85084d7c40c09a87478b3a05a
Parents: 08d69d9
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Nov 6 17:55:51 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Nov 6 17:55:51 2018 -0600
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba1f9d66/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index aeda9cc..c222685 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -910,6 +910,8 @@ public class ShuffleHandler extends AuxiliaryService {
@Override
public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent evt)
throws Exception {
+ super.channelOpen(ctx, evt);
+
if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) {
LOG.info(String.format("Current number of shuffle connections (%d) is " +
"greater than or equal to the max allowed shuffle connections (%d)",
@@ -925,8 +927,6 @@ public class ShuffleHandler extends AuxiliaryService {
return;
}
accepted.add(evt.getChannel());
- super.channelOpen(ctx, evt);
-
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/29] hadoop git commit: YARN-8988. Reduce the verbose log on RM
heartbeat path when distributed node-attributes is enabled. Contributed by
Tao Yang.
Posted by ae...@apache.org.
YARN-8988. Reduce the verbose log on RM heartbeat path when distributed node-attributes is enabled. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e1bbf7dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e1bbf7dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e1bbf7dc
Branch: refs/heads/HDDS-4
Commit: e1bbf7dcdfc30a61a2b10bef09c59ff17d290488
Parents: f8c72d7
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Nov 8 17:47:18 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Nov 8 17:47:18 2018 +0800
----------------------------------------------------------------------
.../resourcemanager/nodelabels/NodeAttributesManagerImpl.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1bbf7dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
index e524788..83c5983 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -221,8 +221,10 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
// Notify RM
if (rmContext != null && rmContext.getDispatcher() != null) {
- LOG.info("Updated NodeAttribute event to RM:" + newNodeToAttributesMap
- .values());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updated NodeAttribute event to RM:"
+ + newNodeToAttributesMap.values());
+ }
rmContext.getDispatcher().getEventHandler().handle(
new NodeAttributesUpdateSchedulerEvent(newNodeToAttributesMap));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/29] hadoop git commit: YARN-8865. RMStateStore contains large
number of expired RMDelegationToken. Contributed by Wilfred Spiegelenburg
Posted by ae...@apache.org.
YARN-8865. RMStateStore contains large number of expired RMDelegationToken. Contributed by Wilfred Spiegelenburg
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab6aa4c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab6aa4c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab6aa4c7
Branch: refs/heads/HDDS-4
Commit: ab6aa4c7265db5bcbb446c2f779289023d454b81
Parents: 49412a1
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Nov 6 08:40:59 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Nov 6 08:40:59 2018 -0600
----------------------------------------------------------------------
.../AbstractDelegationTokenSecretManager.java | 12 ++-
.../hs/TestJHSDelegationTokenSecretManager.java | 78 +++++++++++++++++---
.../security/TestRMDelegationTokens.java | 63 ++++++++++++++++
3 files changed, 140 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab6aa4c7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index 7593b4e..9259d87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
@@ -300,7 +300,8 @@ extends AbstractDelegationTokenIdentifier>
/**
* This method is intended to be used for recovering persisted delegation
- * tokens
+ * tokens. Tokens that have an unknown <code>DelegationKey</code> are
+ * marked as expired and automatically cleaned up.
* This method must be called before this secret manager is activated (before
* startThreads() is called)
* @param identifier identifier read from persistent storage
@@ -316,12 +317,15 @@ extends AbstractDelegationTokenIdentifier>
}
int keyId = identifier.getMasterKeyId();
DelegationKey dKey = allKeys.get(keyId);
+ byte[] password = null;
if (dKey == null) {
- LOG.warn("No KEY found for persisted identifier "
+ LOG.warn("No KEY found for persisted identifier, expiring stored token "
+ formatTokenId(identifier));
- return;
+ // make sure the token is expired
+ renewDate = 0L;
+ } else {
+ password = createPassword(identifier.getBytes(), dKey.getKey());
}
- byte[] password = createPassword(identifier.getBytes(), dKey.getKey());
if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
setDelegationTokenSeqNum(identifier.getSequenceNumber());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab6aa4c7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java
index f41bb3a..64715fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJHSDelegationTokenSecretManager.java
@@ -21,12 +21,12 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Supplier;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
@@ -34,17 +34,21 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
import org.junit.Test;
public class TestJHSDelegationTokenSecretManager {
@Test
- public void testRecovery() throws IOException {
+ public void testRecovery() throws Exception {
Configuration conf = new Configuration();
HistoryServerStateStoreService store =
new HistoryServerMemStateStoreService();
store.init(conf);
store.start();
+ Map<MRDelegationTokenIdentifier, Long> tokenState =
+ ((HistoryServerMemStateStoreService) store).state.getTokenState();
JHSDelegationTokenSecretManagerForTest mgr =
new JHSDelegationTokenSecretManagerForTest(store);
mgr.startThreads();
@@ -63,9 +67,15 @@ public class TestJHSDelegationTokenSecretManager {
DelegationKey[] keys = mgr.getAllKeys();
long tokenRenewDate1 = mgr.getAllTokens().get(tokenId1).getRenewDate();
long tokenRenewDate2 = mgr.getAllTokens().get(tokenId2).getRenewDate();
- mgr.stopThreads();
+ // Make sure we stored the tokens
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return tokenState.size() == 2;
+ }
+ }, 10, 2000);
+
+ stopAndCleanSecretManager(mgr);
- mgr = new JHSDelegationTokenSecretManagerForTest(store);
mgr.recover(store.loadState());
List<DelegationKey> recoveredKeys = Arrays.asList(mgr.getAllKeys());
for (DelegationKey key : keys) {
@@ -106,27 +116,77 @@ public class TestJHSDelegationTokenSecretManager {
}
// Succeed to cancel with full principal
mgr.cancelToken(tokenFull, tokenIdFull.getOwner().toString());
+ // Make sure we removed the stored token
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return tokenState.size() == 2;
+ }
+ }, 10, 2000);
long tokenRenewDate3 = mgr.getAllTokens().get(tokenId3).getRenewDate();
- mgr.stopThreads();
+ stopAndCleanSecretManager(mgr);
- mgr = new JHSDelegationTokenSecretManagerForTest(store);
mgr.recover(store.loadState());
assertFalse("token1 should be missing",
mgr.getAllTokens().containsKey(tokenId1));
assertTrue("token2 missing", mgr.getAllTokens().containsKey(tokenId2));
- assertEquals("token2 renew date", tokenRenewDate2,
+ assertEquals("token2 renew date incorrect", tokenRenewDate2,
mgr.getAllTokens().get(tokenId2).getRenewDate());
- assertTrue("token3 missing", mgr.getAllTokens().containsKey(tokenId3));
+ assertTrue("token3 missing from manager",
+ mgr.getAllTokens().containsKey(tokenId3));
assertEquals("token3 renew date", tokenRenewDate3,
mgr.getAllTokens().get(tokenId3).getRenewDate());
mgr.startThreads();
mgr.verifyToken(tokenId2, token2.getPassword());
mgr.verifyToken(tokenId3, token3.getPassword());
- mgr.stopThreads();
+ // Set an unknown key ID: token should not be restored
+ tokenId3.setMasterKeyId(1000);
+ // Update renewal date to check the store write
+ mgr.updateStoredToken(tokenId3, tokenRenewDate3 + 5000);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return tokenState.get(tokenId3).equals(tokenRenewDate3 + 5000);
+ }
+ }, 10, 2000);
+ stopAndCleanSecretManager(mgr);
+
+ // Store should contain token but manager should not
+ Assert.assertTrue("Store does not contain token3",
+ tokenState.containsKey(tokenId3));
+ Assert.assertFalse("Store does not contain token3",
+ mgr.getAllTokens().containsKey(tokenId3));
+ // Recover to load the token into the manager; renew date is set to 0
+ mgr.recover(store.loadState());
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return mgr.getAllTokens().get(tokenId3).getRenewDate() == 0L;
+ }
+ }, 10, 2000);
+ // Start the removal threads: cleanup manager and store
+ mgr.startThreads();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return !mgr.getAllTokens().containsKey(tokenId3);
+ }
+ }, 10, 2000);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return !tokenState.containsKey(tokenId3);
+ }
+ }, 10, 2000);
}
+ private void stopAndCleanSecretManager(
+ JHSDelegationTokenSecretManagerForTest mgr) {
+ mgr.stopThreads();
+ mgr.reset();
+ Assert.assertEquals("Secret manager should not contain keys",
+ mgr.getAllKeys().length, 0);
+ Assert.assertEquals("Secret manager should not contain tokens",
+ mgr.getAllTokens().size(), 0);
+ }
+
private static class JHSDelegationTokenSecretManagerForTest
extends JHSDelegationTokenSecretManager {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab6aa4c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
index aae86b0..5b55d68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
@@ -26,13 +26,16 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.base.Supplier;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -174,6 +177,66 @@ public class TestRMDelegationTokens {
rm1.stop();
}
+ // Test removing token without key from state-store.
+ @Test(timeout = 15000)
+ public void testUnknownKeyTokensOnRecover() throws Exception {
+ final int masterID = 1234;
+ final int sequenceNumber = 1000;
+
+ MemoryRMStateStore memStore = new MockMemoryRMStateStore();
+ memStore.init(testConf);
+ // Need RM to get the secret manager and call recover
+ MockRM rm1 = new MyMockRM(testConf, memStore);
+ rm1.start();
+ RMDelegationTokenSecretManager dtSecretManager =
+ rm1.getRMContext().getRMDelegationTokenSecretManager();
+ RMState rmState = memStore.getState();
+ // short cut to generate a basic token with unknown key
+ RMDelegationTokenIdentifier rmDT = new RMDelegationTokenIdentifier(
+ new Text("owner"), new Text("renewer"), new Text("realuser"));
+ // set a master key which is not used
+ rmDT.setMasterKeyId(masterID);
+ rmDT.setSequenceNumber(sequenceNumber);
+ final long tokenTime = Time.now() + 60000;
+ rmDT.setMaxDate(tokenTime);
+ dtSecretManager.storeNewToken(rmDT, tokenTime);
+ // give it time to process
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return rmState.getRMDTSecretManagerState().getTokenState().
+ containsKey(rmDT);
+ }
+ }, 10, 2000);
+ // Cannot recover while running: stop and clear
+ dtSecretManager.stopThreads();
+ dtSecretManager.reset();
+ Assert.assertEquals("Secret manager should have no tokens",
+ dtSecretManager.getAllTokens().size(), 0);
+ Assert.assertEquals("Secret manager should have no keys",
+ dtSecretManager.getAllMasterKeys().size(), 0);
+ dtSecretManager.recover(rmState);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return
+ rmState.getRMDTSecretManagerState().getTokenState().
+ containsKey(rmDT);
+ }
+ }, 10, 2000);
+ Assert.assertEquals("Token should have been expired but is not", 0L,
+ dtSecretManager.getRenewDate(rmDT));
+ // The remover thread should immediately do its work,
+ // still give it some time to process
+ dtSecretManager.startThreads();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return
+ !rmState.getRMDTSecretManagerState().getTokenState().
+ containsKey(rmDT);
+ }
+ }, 10, 2000);
+ rm1.stop();
+ }
+
class MyMockRM extends TestSecurityMockRM {
public MyMockRM(Configuration conf, RMStateStore store) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/29] hadoop git commit: YARN-8233. NPE in
CapacityScheduler#tryCommit when handling allocate/reserve proposal whose
allocatedOrReservedContainer is null. Contributed by Tao Yang.
Posted by ae...@apache.org.
YARN-8233. NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/951c98f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/951c98f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/951c98f8
Branch: refs/heads/HDDS-4
Commit: 951c98f89059d64fda8456366f680eff4a7a6785
Parents: ba1f9d6
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Nov 7 11:17:35 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Nov 7 11:17:35 2018 +0900
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 86 +++++++++++++-------
.../TestCapacitySchedulerAsyncScheduling.java | 83 +++++++++++++++++++
2 files changed, 141 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951c98f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 5d7f1ba..e604b81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2771,7 +2771,11 @@ public class CapacityScheduler extends
.getContainersToKill().isEmpty()) {
list = new ArrayList<>();
for (RMContainer rmContainer : csAssignment.getContainersToKill()) {
- list.add(getSchedulerContainer(rmContainer, false));
+ SchedulerContainer schedulerContainer =
+ getSchedulerContainer(rmContainer, false);
+ if (schedulerContainer != null) {
+ list.add(schedulerContainer);
+ }
}
}
@@ -2779,10 +2783,16 @@ public class CapacityScheduler extends
if (null == list) {
list = new ArrayList<>();
}
- list.add(
- getSchedulerContainer(csAssignment.getExcessReservation(), false));
+ SchedulerContainer schedulerContainer =
+ getSchedulerContainer(csAssignment.getExcessReservation(), false);
+ if (schedulerContainer != null) {
+ list.add(schedulerContainer);
+ }
}
+ if (list != null && list.isEmpty()) {
+ list = null;
+ }
return list;
}
@@ -2867,11 +2877,15 @@ public class CapacityScheduler extends
((RMContainerImpl)rmContainer).setAllocationTags(
new HashSet<>(schedulingRequest.getAllocationTags()));
- allocated = new ContainerAllocationProposal<>(
- getSchedulerContainer(rmContainer, true),
- null, null, NodeType.NODE_LOCAL, NodeType.NODE_LOCAL,
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
- resource);
+ SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
+ schedulerContainer = getSchedulerContainer(rmContainer, true);
+ if (schedulerContainer == null) {
+ allocated = null;
+ } else {
+ allocated = new ContainerAllocationProposal<>(schedulerContainer,
+ null, null, NodeType.NODE_LOCAL, NodeType.NODE_LOCAL,
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, resource);
+ }
}
if (null != allocated) {
@@ -2901,16 +2915,27 @@ public class CapacityScheduler extends
csAssignment.getAssignmentInformation().getAllocationDetails();
if (!allocations.isEmpty()) {
RMContainer rmContainer = allocations.get(0).rmContainer;
- allocated = new ContainerAllocationProposal<>(
- getSchedulerContainer(rmContainer, true),
- getSchedulerContainersToRelease(csAssignment),
- getSchedulerContainer(csAssignment.getFulfilledReservedContainer(),
- false), csAssignment.getType(),
- csAssignment.getRequestLocalityType(),
- csAssignment.getSchedulingMode() != null ?
- csAssignment.getSchedulingMode() :
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
- csAssignment.getResource());
+ SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
+ schedulerContainer = getSchedulerContainer(rmContainer, true);
+ if (schedulerContainer == null) {
+ allocated = null;
+ // Decrease unconfirmed resource if app is alive
+ FiCaSchedulerApp app = getApplicationAttempt(
+ rmContainer.getApplicationAttemptId());
+ if (app != null) {
+ app.decUnconfirmedRes(rmContainer.getAllocatedResource());
+ }
+ } else {
+ allocated = new ContainerAllocationProposal<>(schedulerContainer,
+ getSchedulerContainersToRelease(csAssignment),
+ getSchedulerContainer(
+ csAssignment.getFulfilledReservedContainer(), false),
+ csAssignment.getType(), csAssignment.getRequestLocalityType(),
+ csAssignment.getSchedulingMode() != null ?
+ csAssignment.getSchedulingMode() :
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
+ csAssignment.getResource());
+ }
}
// Reserved something
@@ -2918,16 +2943,21 @@ public class CapacityScheduler extends
csAssignment.getAssignmentInformation().getReservationDetails();
if (!reservation.isEmpty()) {
RMContainer rmContainer = reservation.get(0).rmContainer;
- reserved = new ContainerAllocationProposal<>(
- getSchedulerContainer(rmContainer, false),
- getSchedulerContainersToRelease(csAssignment),
- getSchedulerContainer(csAssignment.getFulfilledReservedContainer(),
- false), csAssignment.getType(),
- csAssignment.getRequestLocalityType(),
- csAssignment.getSchedulingMode() != null ?
- csAssignment.getSchedulingMode() :
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
- csAssignment.getResource());
+ SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
+ schedulerContainer = getSchedulerContainer(rmContainer, false);
+ if (schedulerContainer == null) {
+ reserved = null;
+ } else {
+ reserved = new ContainerAllocationProposal<>(schedulerContainer,
+ getSchedulerContainersToRelease(csAssignment),
+ getSchedulerContainer(
+ csAssignment.getFulfilledReservedContainer(), false),
+ csAssignment.getType(), csAssignment.getRequestLocalityType(),
+ csAssignment.getSchedulingMode() != null ?
+ csAssignment.getSchedulingMode() :
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
+ csAssignment.getResource());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951c98f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 840d30d..67c504d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -56,8 +56,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerA
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
@@ -843,6 +846,86 @@ public class TestCapacitySchedulerAsyncScheduling {
return new ResourceCommitRequest(allocateProposals, null, null);
}
+ @Test(timeout = 30000)
+ public void testReturnNullWhenGetSchedulerContainer() throws Exception {
+ // disable async-scheduling for simulating complex scenario
+ Configuration disableAsyncConf = new Configuration(conf);
+ disableAsyncConf.setBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+ // init RM & NMs
+ final MockRM rm = new MockRM(disableAsyncConf);
+ rm.start();
+ final MockNM nm1 = rm.registerNode("192.168.0.1:1234", 8 * GB);
+ final MockNM nm2 = rm.registerNode("192.168.0.2:2234", 8 * GB);
+ rm.drainEvents();
+ CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
+ SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId());
+ RMNode rmNode1 = cs.getNode(nm1.getNodeId()).getRMNode();
+ SchedulerNode sn2 = cs.getSchedulerNode(nm2.getNodeId());
+
+ // launch app1-am on nm1
+ RMApp app1 = rm.submitApp(1 * GB, "app1", "user", null, false, "default",
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+ // app2 asks 1 * 1G container
+ am1.allocate(ImmutableList.of(ResourceRequest
+ .newInstance(Priority.newInstance(0), "*",
+ Resources.createResource(1 * GB), 1)), null);
+ RMContainer amContainer = cs.getRMContainer(
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1));
+
+ // spy CapacityScheduler
+ final CapacityScheduler spyCs = Mockito.spy(cs);
+ // hook CapacityScheduler#submitResourceCommitRequest
+ List<CSAssignment> assignmentSnapshots = new ArrayList<>();
+ Mockito.doAnswer(new Answer<Object>() {
+ public Boolean answer(InvocationOnMock invocation) throws Exception {
+ CSAssignment assignment = (CSAssignment) invocation.getArguments()[1];
+ if (cs.getNode(nm1.getNodeId()) != null) {
+ // decommission nm1 for first allocation on nm1
+ cs.getRMContext().getDispatcher().getEventHandler().handle(
+ new RMNodeEvent(nm1.getNodeId(), RMNodeEventType.DECOMMISSION));
+ rm.drainEvents();
+ Assert.assertEquals(NodeState.DECOMMISSIONED, rmNode1.getState());
+ Assert.assertNull(cs.getNode(nm1.getNodeId()));
+ assignmentSnapshots.add(assignment);
+ } else {
+ // add am container on nm1 to containersToKill
+ // for second allocation on nm2
+ assignment.setContainersToKill(ImmutableList.of(amContainer));
+ }
+ // check no NPE in actual submit, before YARN-8233 will throw NPE
+ cs.submitResourceCommitRequest((Resource) invocation.getArguments()[0],
+ assignment);
+ return false;
+ }
+ }).when(spyCs).submitResourceCommitRequest(Mockito.any(Resource.class),
+ Mockito.any(CSAssignment.class));
+
+ // allocation on nm1, test return null when get scheduler container
+ CandidateNodeSet<FiCaSchedulerNode> candidateNodeSet =
+ new SimpleCandidateNodeSet(sn1);
+ spyCs.allocateContainersToNode(candidateNodeSet, false);
+ // make sure unconfirmed resource is decreased correctly
+ Assert.assertTrue(spyCs.getApplicationAttempt(am1.getApplicationAttemptId())
+ .hasPendingResourceRequest(RMNodeLabelsManager.NO_LABEL,
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
+
+ // allocation on nm2,
+ // test return null when get scheduler container to release
+ candidateNodeSet =
+ new SimpleCandidateNodeSet(sn2);
+ spyCs.allocateContainersToNode(candidateNodeSet, false);
+ // make sure unconfirmed resource is decreased correctly
+ Assert.assertTrue(spyCs.getApplicationAttempt(am1.getApplicationAttemptId())
+ .hasPendingResourceRequest(RMNodeLabelsManager.NO_LABEL,
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
+
+ rm.stop();
+ }
+
private void keepNMHeartbeat(List<MockNM> mockNMs, int interval) {
if (nmHeartbeatThread != null) {
nmHeartbeatThread.setShouldStop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/29] hadoop git commit: HDFS-14048. DFSOutputStream close() throws
exception on subsequent call after DataNode restart. Contributed by Erik
Krogen.
Posted by ae...@apache.org.
HDFS-14048. DFSOutputStream close() throws exception on subsequent call after DataNode restart. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/887244de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/887244de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/887244de
Branch: refs/heads/HDDS-4
Commit: 887244de4adebe27693ed4ad3296a6f700cfa8c1
Parents: 00a67f7
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Nov 6 11:18:15 2018 -0800
Committer: Inigo Goiri <in...@apache.org>
Committed: Tue Nov 6 11:18:15 2018 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java | 1 +
.../apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java | 2 ++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/887244de/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index c1473dd..4c733bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -1781,6 +1781,7 @@ class DataStreamer extends Daemon {
blockStream = out;
result = true; // success
errorState.resetInternalError();
+ lastException.clear();
// remove all restarting nodes from failed nodes list
failed.removeAll(restartingNodes);
restartingNodes.clear();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/887244de/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
index 3f8c7f7..465a083 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
@@ -436,6 +436,8 @@ public class TestClientProtocolForPipelineRecovery {
0, out.getStreamer().getPipelineRecoveryCount());
out.write(1);
out.close();
+ // Ensure that subsequent closes are idempotent and do not throw errors
+ out.close();
} finally {
if (cluster != null) {
cluster.shutdown();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/29] hadoop git commit: HDDS-737. Introduce Incremental Container
Report. Contributed by Nanda kumar.
Posted by ae...@apache.org.
HDDS-737. Introduce Incremental Container Report.
Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c80f753b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c80f753b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c80f753b
Branch: refs/heads/HDDS-4
Commit: c80f753b0e95eb722a972f836c1e4d16fd823434
Parents: e1bbf7d
Author: Nanda kumar <na...@apache.org>
Authored: Thu Nov 8 18:33:38 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Thu Nov 8 18:33:38 2018 +0530
----------------------------------------------------------------------
.../hdds/scm/container/ContainerInfo.java | 8 +
.../statemachine/DatanodeStateMachine.java | 17 +-
.../common/statemachine/StateContext.java | 13 +-
.../CloseContainerCommandHandler.java | 73 +++--
.../states/endpoint/HeartbeatEndpointTask.java | 12 +-
.../StorageContainerDatanodeProtocol.proto | 2 +-
.../scm/command/CommandStatusReportHandler.java | 4 -
.../container/CloseContainerEventHandler.java | 27 --
.../scm/container/CloseContainerWatcher.java | 101 -------
.../hdds/scm/container/ContainerManager.java | 11 -
.../hdds/scm/container/ContainerReplica.java | 30 +-
.../scm/container/ContainerReportHandler.java | 202 +++++++++----
.../IncrementalContainerReportHandler.java | 98 +++++++
.../hdds/scm/container/SCMContainerManager.java | 107 -------
.../scm/container/states/ContainerStateMap.java | 7 -
.../hadoop/hdds/scm/events/SCMEvents.java | 34 +--
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 37 ++-
.../hadoop/hdds/scm/node/NewNodeHandler.java | 16 +-
.../hadoop/hdds/scm/node/NodeManager.java | 50 +---
.../hadoop/hdds/scm/node/NodeReportHandler.java | 2 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 69 +----
.../hadoop/hdds/scm/node/SCMNodeManager.java | 66 +----
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 2 +-
.../hdds/scm/node/states/NodeStateMap.java | 67 ++---
.../server/SCMDatanodeHeartbeatDispatcher.java | 17 +-
.../scm/server/SCMDatanodeProtocolServer.java | 6 +-
.../scm/server/StorageContainerManager.java | 21 +-
.../command/TestCommandStatusReportHandler.java | 14 -
.../hdds/scm/container/MockNodeManager.java | 66 ++---
.../container/TestContainerReportHandler.java | 15 +-
.../container/TestContainerStateManager.java | 3 +
.../scm/container/TestSCMContainerManager.java | 107 +------
.../replication/TestReplicationManager.java | 12 +-
.../hdds/scm/node/TestDeadNodeHandler.java | 24 +-
.../container/TestCloseContainerWatcher.java | 289 -------------------
.../ozone/container/common/TestEndPoint.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 55 +---
.../TestContainerStateManagerIntegration.java | 11 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 4 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 4 +-
.../commandhandler/TestBlockDeletion.java | 6 +-
41 files changed, 558 insertions(+), 1157 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
index 5a9484a..edfa0f9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
@@ -158,10 +158,18 @@ public class ContainerInfo implements Comparator<ContainerInfo>,
return usedBytes;
}
+ public void setUsedBytes(long value) {
+ usedBytes = value;
+ }
+
public long getNumberOfKeys() {
return numberOfKeys;
}
+ public void setNumberOfKeys(long value) {
+ numberOfKeys = value;
+ }
+
public long getDeleteTransactionId() {
return deleteTransactionId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 4768cf8..12c33ff 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -168,10 +168,14 @@ public class DatanodeStateMachine implements Closeable {
TimeUnit.MILLISECONDS);
now = Time.monotonicNow();
if (now < nextHB.get()) {
- Thread.sleep(nextHB.get() - now);
+ if(!Thread.interrupted()) {
+ Thread.sleep(nextHB.get() - now);
+ }
}
} catch (InterruptedException e) {
- // Ignore this exception.
+ // Some one has sent interrupt signal, this could be because
+ // 1. Trigger heartbeat immediately
+ // 2. Shutdown has be initiated.
} catch (Exception e) {
LOG.error("Unable to finish the execution.", e);
}
@@ -325,6 +329,15 @@ public class DatanodeStateMachine implements Closeable {
}
/**
+ * Calling this will immediately trigger a heartbeat to the SCMs.
+ * This heartbeat will also include all the reports which are ready to
+ * be sent by datanode.
+ */
+ public void triggerHeartbeat() {
+ stateMachineThread.interrupt();
+ }
+
+ /**
* Waits for DatanodeStateMachine to exit.
*
* @throws InterruptedException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 12c196b..e928824 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -447,6 +447,11 @@ public class StateContext {
* @param cmd - {@link SCMCommand}.
*/
public void addCmdStatus(SCMCommand cmd) {
+ if (cmd.getType().equals(Type.closeContainerCommand)) {
+ // We will be removing CommandStatus completely.
+ // As a first step, removed it for CloseContainerCommand.
+ return;
+ }
CommandStatusBuilder statusBuilder;
if (cmd.getType() == Type.deleteBlocksCommand) {
statusBuilder = new DeleteBlockCommandStatusBuilder();
@@ -469,14 +474,6 @@ public class StateContext {
}
/**
- * Remove object from cache in StateContext#cmdStatusMap.
- *
- */
- public void removeCommandStatus(Long cmdId) {
- cmdStatusMap.remove(cmdId);
- }
-
- /**
* Updates status of a pending status command.
* @param cmdId command id
* @param cmdStatusUpdater Consumer to update command status.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index 2c3db61..591fe56 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.CloseContainerCommandProto;
@@ -63,62 +65,55 @@ public class CloseContainerCommandHandler implements CommandHandler {
StateContext context, SCMConnectionManager connectionManager) {
LOG.debug("Processing Close Container command.");
invocationCount++;
- cmdExecuted = false;
long startTime = Time.monotonicNow();
// TODO: define this as INVALID_CONTAINER_ID in HddsConsts.java (TBA)
long containerID = -1;
try {
-
- CloseContainerCommandProto
- closeContainerProto =
- CloseContainerCommandProto
- .parseFrom(command.getProtoBufMessage());
+ CloseContainerCommandProto closeContainerProto =
+ CloseContainerCommandProto.parseFrom(command.getProtoBufMessage());
containerID = closeContainerProto.getContainerID();
- if (container.getContainerSet().getContainer(containerID)
+ // CloseContainer operation is idempotent, if the container is already
+ // closed, then do nothing.
+ if (!container.getContainerSet().getContainer(containerID)
.getContainerData().isClosed()) {
- LOG.debug("Container {} is already closed", containerID);
- // It might happen that the where the first attempt of closing the
- // container failed with NOT_LEADER_EXCEPTION. In such cases, SCM will
- // retry to check the container got really closed via Ratis.
- // In such cases of the retry attempt, if the container is already
- // closed via Ratis, we should just return.
- cmdExecuted = true;
- return;
- }
- HddsProtos.PipelineID pipelineID = closeContainerProto.getPipelineID();
- HddsProtos.ReplicationType replicationType =
- closeContainerProto.getReplicationType();
+ LOG.debug("Closing container {}.", containerID);
+ HddsProtos.PipelineID pipelineID = closeContainerProto.getPipelineID();
+ HddsProtos.ReplicationType replicationType =
+ closeContainerProto.getReplicationType();
- ContainerProtos.ContainerCommandRequestProto.Builder request =
- ContainerProtos.ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.CloseContainer);
- request.setContainerID(containerID);
- request.setCloseContainer(
- ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
- request.setTraceID(UUID.randomUUID().toString());
- request.setDatanodeUuid(
- context.getParent().getDatanodeDetails().getUuidString());
- // submit the close container request for the XceiverServer to handle
- container.submitContainerRequest(
- request.build(), replicationType, pipelineID);
+ ContainerProtos.ContainerCommandRequestProto.Builder request =
+ ContainerProtos.ContainerCommandRequestProto.newBuilder();
+ request.setCmdType(ContainerProtos.Type.CloseContainer);
+ request.setContainerID(containerID);
+ request.setCloseContainer(
+ ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
+ request.setTraceID(UUID.randomUUID().toString());
+ request.setDatanodeUuid(
+ context.getParent().getDatanodeDetails().getUuidString());
+ // submit the close container request for the XceiverServer to handle
+ container.submitContainerRequest(
+ request.build(), replicationType, pipelineID);
+ // Since the container is closed, we trigger an ICR
+ IncrementalContainerReportProto icr = IncrementalContainerReportProto
+ .newBuilder()
+ .addReport(container.getContainerSet()
+ .getContainer(containerID).getContainerReport())
+ .build();
+ context.addReport(icr);
+ context.getParent().triggerHeartbeat();
+ }
} catch (Exception e) {
if (e instanceof NotLeaderException) {
// If the particular datanode is not the Ratis leader, the close
// container command will not be executed by the follower but will be
// executed by Ratis stateMachine transactions via leader to follower.
// There can also be case where the datanode is in candidate state.
- // In these situations, NotLeaderException is thrown. Remove the status
- // from cmdStatus Map here so that it will be retried only by SCM if the
- // leader could not not close the container after a certain time.
- context.removeCommandStatus(containerID);
- LOG.info(e.getLocalizedMessage());
+ // In these situations, NotLeaderException is thrown.
+ LOG.info("Follower cannot close the container {}.", containerID);
} else {
LOG.error("Can't close container " + containerID, e);
- cmdExecuted = false;
}
} finally {
- updateCommandStatus(context, command,
- (cmdStatus) -> cmdStatus.setStatus(cmdExecuted), LOG);
long endTime = Time.monotonicNow();
totalTime += endTime - startTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 4fd72ec..0c0f1af 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -141,7 +141,9 @@ public class HeartbeatEndpointTask
rpcEndpoint.zeroMissedCount();
} catch (IOException ex) {
// put back the reports which failed to be sent
- putBackReports(requestBuilder);
+ if (requestBuilder != null) {
+ putBackReports(requestBuilder);
+ }
rpcEndpoint.logIfNeeded(ex);
} finally {
rpcEndpoint.unlock();
@@ -159,10 +161,10 @@ public class HeartbeatEndpointTask
reports.add(requestBuilder.getNodeReport());
}
if (requestBuilder.getCommandStatusReportsCount() != 0) {
- for (GeneratedMessage msg : requestBuilder
- .getCommandStatusReportsList()) {
- reports.add(msg);
- }
+ reports.addAll(requestBuilder.getCommandStatusReportsList());
+ }
+ if (requestBuilder.getIncrementalContainerReportCount() != 0) {
+ reports.addAll(requestBuilder.getIncrementalContainerReportList());
}
context.putBackReports(reports);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 4ddb7b2..33ea307 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -80,7 +80,7 @@ message SCMHeartbeatRequestProto {
required DatanodeDetailsProto datanodeDetails = 1;
optional NodeReportProto nodeReport = 2;
optional ContainerReportsProto containerReport = 3;
- optional IncrementalContainerReportProto incrementalContainerReport = 4;
+ repeated IncrementalContainerReportProto incrementalContainerReport = 4;
repeated CommandStatusReportsProto commandStatusReports = 5;
optional ContainerActionsProto containerActions = 6;
optional PipelineActionsProto pipelineActions = 7;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
index c0de382..53dfc5a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
@@ -58,10 +58,6 @@ public class CommandStatusReportHandler implements
publisher.fireEvent(SCMEvents.REPLICATION_STATUS, new
ReplicationStatus(cmdStatus));
break;
- case closeContainerCommand:
- publisher.fireEvent(SCMEvents.CLOSE_CONTAINER_STATUS, new
- CloseContainerStatus(cmdStatus));
- break;
case deleteBlocksCommand:
if (cmdStatus.getStatus() == CommandStatus.Status.EXECUTED) {
publisher.fireEvent(SCMEvents.DELETE_BLOCK_STATUS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
index 69574a9..9796a96 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
@@ -22,14 +22,12 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.hdds.server.events.IdentifiableEventPayload;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
-import static org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ;
/**
* In case of a node failure, volume failure, volume out of spapce, node
@@ -128,32 +126,7 @@ public class CloseContainerEventHandler implements EventHandler<ContainerID> {
new CommandForDatanode<>(node.getUuid(), closeContainerCommand))
.forEach(command -> publisher.fireEvent(DATANODE_COMMAND, command));
- publisher.fireEvent(CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(containerID));
-
LOG.trace("Issuing {} on Pipeline {} for container", closeContainerCommand,
pipeline, containerID);
}
-
- /**
- * Class to create retryable event. Prevents redundant requests for same
- * container Id.
- */
- public static class CloseContainerRetryableReq implements
- IdentifiableEventPayload {
-
- private ContainerID containerID;
- public CloseContainerRetryableReq(ContainerID containerID) {
- this.containerID = containerID;
- }
-
- public ContainerID getContainerID() {
- return containerID;
- }
-
- @Override
- public long getId() {
- return containerID.getId();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerWatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerWatcher.java
deleted file mode 100644
index 4593c1f..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerWatcher.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * <p>Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container;
-
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.CommandStatus.Status;
-import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
- .CloseContainerStatus;
-
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.Event;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.hdds.server.events.EventWatcher;
-import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler
- .CloseContainerRetryableReq;
-import org.apache.hadoop.ozone.lease.LeaseManager;
-import org.apache.hadoop.ozone.lease.LeaseNotFoundException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * This watcher will watch for CLOSE_CONTAINER_STATUS events fired from
- * CommandStatusReport. If required it will re-trigger CloseContainer command
- * for DataNodes to CloseContainerEventHandler.
- */
-public class CloseContainerWatcher extends
- EventWatcher<CloseContainerRetryableReq, CloseContainerStatus> {
-
- public static final Logger LOG =
- LoggerFactory.getLogger(CloseContainerWatcher.class);
- private final ContainerManager containerManager;
-
- public CloseContainerWatcher(Event<CloseContainerRetryableReq> startEvent,
- Event<CloseContainerStatus> completionEvent,
- LeaseManager<Long> leaseManager, ContainerManager containerManager) {
- super(startEvent, completionEvent, leaseManager);
- this.containerManager = containerManager;
- }
-
- @Override
- protected void onTimeout(EventPublisher publisher,
- CloseContainerRetryableReq payload) {
- // Let CloseContainerEventHandler handle this message.
- this.resendEventToHandler(payload.getId(), publisher);
- }
-
- @Override
- protected void onFinished(EventPublisher publisher,
- CloseContainerRetryableReq payload) {
- LOG.trace("CloseContainerCommand for containerId: {} executed ", payload
- .getContainerID().getId());
- }
-
- @Override
- protected synchronized void handleCompletion(CloseContainerStatus status,
- EventPublisher publisher) throws LeaseNotFoundException {
- // If status is PENDING then return without doing anything.
- if(status.getCmdStatus().getStatus().equals(Status.PENDING)){
- return;
- }
-
- CloseContainerRetryableReq closeCont = getTrackedEventbyId(status.getId());
- super.handleCompletion(status, publisher);
- // If status is FAILED then send a msg to Handler to resend the command.
- if (status.getCmdStatus().getStatus().equals(Status.FAILED) && closeCont
- != null) {
- this.resendEventToHandler(closeCont.getId(), publisher);
- }
- }
-
- private void resendEventToHandler(long containerID, EventPublisher
- publisher) {
- try {
- // Check if container is still open
- if (containerManager.getContainer(
- ContainerID.valueof(containerID)).isOpen()) {
- publisher.fireEvent(SCMEvents.CLOSE_CONTAINER,
- ContainerID.valueof(containerID));
- }
- } catch (IOException e) {
- LOG.warn("Error in CloseContainerWatcher while processing event " +
- "for containerId {} ExceptionMsg: ", containerID, e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
index 0a48915..0906ca8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
@@ -16,14 +16,11 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
import java.io.Closeable;
@@ -123,14 +120,6 @@ public interface ContainerManager extends Closeable {
HddsProtos.LifeCycleEvent event) throws IOException;
/**
- * Process container report from Datanode.
- *
- * @param reports Container report
- */
- void processContainerReports(DatanodeDetails datanodeDetails,
- ContainerReportsProto reports) throws IOException;
-
- /**
* Returns the latest list of replicas for given containerId.
*
* @param containerID Container ID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
index f2e80f4..9445fe8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java
@@ -22,6 +22,8 @@ import org.apache.commons.lang3.builder.CompareToBuilder;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import java.util.Optional;
import java.util.UUID;
@@ -32,15 +34,18 @@ import java.util.UUID;
public final class ContainerReplica implements Comparable<ContainerReplica> {
final private ContainerID containerID;
+ final private ContainerReplicaProto.State state;
final private DatanodeDetails datanodeDetails;
final private UUID placeOfBirth;
private Long sequenceId;
- private ContainerReplica(ContainerID containerID, DatanodeDetails datanode,
- UUID originNodeId) {
+ private ContainerReplica(final ContainerID containerID,
+ final ContainerReplicaProto.State state, final DatanodeDetails datanode,
+ final UUID originNodeId) {
this.containerID = containerID;
+ this.state = state;
this.datanodeDetails = datanode;
this.placeOfBirth = originNodeId;
}
@@ -68,6 +73,15 @@ public final class ContainerReplica implements Comparable<ContainerReplica> {
}
/**
+ * Returns the state of this replica.
+ *
+ * @return replica state
+ */
+ public ContainerReplicaProto.State getState() {
+ return state;
+ }
+
+ /**
* Returns the Sequence Id of this replica.
*
* @return Sequence Id
@@ -126,6 +140,7 @@ public final class ContainerReplica implements Comparable<ContainerReplica> {
public static class ContainerReplicaBuilder {
private ContainerID containerID;
+ private ContainerReplicaProto.State state;
private DatanodeDetails datanode;
private UUID placeOfBirth;
private Long sequenceId;
@@ -142,6 +157,12 @@ public final class ContainerReplica implements Comparable<ContainerReplica> {
return this;
}
+ public ContainerReplicaBuilder setContainerState(
+ final ContainerReplicaProto.State containerState) {
+ state = containerState;
+ return this;
+ }
+
/**
* Set DatanodeDetails.
*
@@ -184,9 +205,12 @@ public final class ContainerReplica implements Comparable<ContainerReplica> {
public ContainerReplica build() {
Preconditions.checkNotNull(containerID,
"Container Id can't be null");
+ Preconditions.checkNotNull(state,
+ "Container state can't be null");
Preconditions.checkNotNull(datanode,
"DatanodeDetails can't be null");
- ContainerReplica replica = new ContainerReplica(containerID, datanode,
+ ContainerReplica replica = new ContainerReplica(
+ containerID, state, datanode,
Optional.ofNullable(placeOfBirth).orElse(datanode.getUuid()));
Optional.ofNullable(sequenceId).ifPresent(replica::setSequenceId);
return replica;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index 0cb2f81..c9fe9e4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -17,19 +17,26 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.replication.ReplicationActivityStatus;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
+import org.apache.hadoop.hdds.scm.container.replication
+ .ReplicationActivityStatus;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.states.ReportResult;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.server
+ .SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -47,73 +54,160 @@ public class ContainerReportHandler implements
LoggerFactory.getLogger(ContainerReportHandler.class);
private final NodeManager nodeManager;
+ private final PipelineManager pipelineManager;
private final ContainerManager containerManager;
- private ReplicationActivityStatus replicationStatus;
+ private final ReplicationActivityStatus replicationStatus;
- public ContainerReportHandler(ContainerManager containerManager,
- NodeManager nodeManager,
- ReplicationActivityStatus replicationActivityStatus) {
- Preconditions.checkNotNull(containerManager);
+ public ContainerReportHandler(final NodeManager nodeManager,
+ final PipelineManager pipelineManager,
+ final ContainerManager containerManager,
+ final ReplicationActivityStatus replicationActivityStatus) {
Preconditions.checkNotNull(nodeManager);
+ Preconditions.checkNotNull(pipelineManager);
+ Preconditions.checkNotNull(containerManager);
Preconditions.checkNotNull(replicationActivityStatus);
this.nodeManager = nodeManager;
+ this.pipelineManager = pipelineManager;
this.containerManager = containerManager;
this.replicationStatus = replicationActivityStatus;
}
@Override
- public void onMessage(ContainerReportFromDatanode containerReportFromDatanode,
- EventPublisher publisher) {
+ public void onMessage(final ContainerReportFromDatanode reportFromDatanode,
+ final EventPublisher publisher) {
+
+ final DatanodeDetails datanodeDetails =
+ reportFromDatanode.getDatanodeDetails();
- DatanodeDetails datanodeOrigin =
- containerReportFromDatanode.getDatanodeDetails();
+ final ContainerReportsProto containerReport =
+ reportFromDatanode.getReport();
- ContainerReportsProto containerReport =
- containerReportFromDatanode.getReport();
try {
- //update state in container db and trigger close container events
- containerManager
- .processContainerReports(datanodeOrigin, containerReport);
-
- Set<ContainerID> containerIds = containerReport.getReportsList().stream()
- .map(StorageContainerDatanodeProtocolProtos
- .ContainerReplicaProto::getContainerID)
- .map(ContainerID::new)
- .collect(Collectors.toSet());
-
- ReportResult<ContainerID> reportResult = nodeManager
- .processContainerReport(datanodeOrigin.getUuid(), containerIds);
-
- //we have the report, so we can update the states for the next iteration.
- nodeManager
- .setContainersForDatanode(datanodeOrigin.getUuid(), containerIds);
-
- for (ContainerID containerID : reportResult.getMissingEntries()) {
- final ContainerReplica replica = ContainerReplica.newBuilder()
- .setContainerID(containerID)
- .setDatanodeDetails(datanodeOrigin)
- .build();
- containerManager
- .removeContainerReplica(containerID, replica);
- checkReplicationState(containerID, publisher);
+ final List<ContainerReplicaProto> replicas = containerReport
+ .getReportsList();
+
+ // ContainerIDs which SCM expects this datanode to have.
+ final Set<ContainerID> expectedContainerIDs = nodeManager
+ .getContainers(datanodeDetails);
+
+ // ContainerIDs that this datanode actually has.
+ final Set<ContainerID> actualContainerIDs = replicas.parallelStream()
+ .map(ContainerReplicaProto::getContainerID)
+ .map(ContainerID::valueof).collect(Collectors.toSet());
+
+ // Container replicas which SCM is not aware of.
+ final Set<ContainerID> newReplicas =
+ new HashSet<>(actualContainerIDs);
+ newReplicas.removeAll(expectedContainerIDs);
+
+ // Container replicas which are missing from datanode.
+ final Set<ContainerID> missingReplicas =
+ new HashSet<>(expectedContainerIDs);
+ missingReplicas.removeAll(actualContainerIDs);
+
+ processContainerReplicas(datanodeDetails, replicas, publisher);
+
+ // Remove missing replica from ContainerManager
+ for (ContainerID id : missingReplicas) {
+ try {
+ containerManager.getContainerReplicas(id)
+ .stream()
+ .filter(replica ->
+ replica.getDatanodeDetails().equals(datanodeDetails))
+ .findFirst()
+ .ifPresent(replica -> {
+ try {
+ containerManager.removeContainerReplica(id, replica);
+ } catch (ContainerNotFoundException |
+ ContainerReplicaNotFoundException e) {
+ // This should not happen, but even if it happens, not an
+ // issue
+ }
+ });
+ } catch (ContainerNotFoundException e) {
+ LOG.warn("Cannot remove container replica, container {} not found",
+ id);
+ }
}
- for (ContainerID containerID : reportResult.getNewEntries()) {
- final ContainerReplica replica = ContainerReplica.newBuilder()
- .setContainerID(containerID)
- .setDatanodeDetails(datanodeOrigin)
- .build();
- containerManager.updateContainerReplica(containerID, replica);
- checkReplicationState(containerID, publisher);
- }
+ // Update the latest set of containers for this datanode in NodeManager.
+ nodeManager.setContainers(datanodeDetails, actualContainerIDs);
+
+ // Replicate if needed.
+ newReplicas.forEach(id -> checkReplicationState(id, publisher));
+ missingReplicas.forEach(id -> checkReplicationState(id, publisher));
+
+ } catch (NodeNotFoundException ex) {
+ LOG.error("Received container report from unknown datanode {}",
+ datanodeDetails);
+ }
+
+ }
- } catch (IOException e) {
- //TODO: stop all the replication?
- LOG.error("Error on processing container report from datanode {}",
- datanodeOrigin, e);
+ private void processContainerReplicas(final DatanodeDetails datanodeDetails,
+ final List<ContainerReplicaProto> replicas,
+ final EventPublisher publisher) {
+ final PendingDeleteStatusList pendingDeleteStatusList =
+ new PendingDeleteStatusList(datanodeDetails);
+ for (ContainerReplicaProto replicaProto : replicas) {
+ try {
+ final ContainerID containerID = ContainerID.valueof(
+ replicaProto.getContainerID());
+ final ContainerInfo containerInfo = containerManager
+ .getContainer(containerID);
+ updateContainerState(datanodeDetails, containerInfo,
+ replicaProto, publisher);
+ if (containerInfo.getDeleteTransactionId() >
+ replicaProto.getDeleteTransactionId()) {
+ pendingDeleteStatusList
+ .addPendingDeleteStatus(replicaProto.getDeleteTransactionId(),
+ containerInfo.getDeleteTransactionId(),
+ containerInfo.getContainerID());
+ }
+ } catch (ContainerNotFoundException e) {
+ LOG.error("Received container report for an unknown container {}",
+ replicaProto.getContainerID());
+ }
}
+ if (pendingDeleteStatusList.getNumPendingDeletes() > 0) {
+ publisher.fireEvent(SCMEvents.PENDING_DELETE_STATUS,
+ pendingDeleteStatusList);
+ }
+ }
+ private void updateContainerState(final DatanodeDetails datanodeDetails,
+ final ContainerInfo containerInfo,
+ final ContainerReplicaProto replicaProto,
+ final EventPublisher publisher)
+ throws ContainerNotFoundException {
+
+ final ContainerID id = containerInfo.containerID();
+ final ContainerReplica datanodeContainerReplica = ContainerReplica
+ .newBuilder()
+ .setContainerID(id)
+ .setContainerState(replicaProto.getState())
+ .setDatanodeDetails(datanodeDetails)
+ .build();
+ // TODO: Add bcsid and origin datanode to replica.
+
+ final ContainerReplica scmContainerReplica = containerManager
+ .getContainerReplicas(id)
+ .stream()
+ .filter(replica ->
+ replica.getDatanodeDetails().equals(datanodeDetails))
+ .findFirst().orElse(null);
+
+ // This is an in-memory update.
+ containerManager.updateContainerReplica(id, datanodeContainerReplica);
+ containerInfo.setUsedBytes(replicaProto.getUsed());
+ containerInfo.setNumberOfKeys(replicaProto.getKeyCount());
+
+ // Check if there is state change in container replica.
+ if (scmContainerReplica == null ||
+ scmContainerReplica.getState() != datanodeContainerReplica.getState()) {
+ //TODO: Handler replica state change.
+ }
}
private void checkReplicationState(ContainerID containerID,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
new file mode 100644
index 0000000..e07ee73
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/IncrementalContainerReportHandler.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.server
+ .SCMDatanodeHeartbeatDispatcher.IncrementalContainerReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Handles incremental container reports from datanode.
+ */
+public class IncrementalContainerReportHandler implements
+ EventHandler<IncrementalContainerReportFromDatanode> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(IncrementalContainerReportHandler.class);
+
+ private final PipelineManager pipelineManager;
+ private final ContainerManager containerManager;
+
+ public IncrementalContainerReportHandler(
+ final PipelineManager pipelineManager,
+ final ContainerManager containerManager) {
+ Preconditions.checkNotNull(pipelineManager);
+ Preconditions.checkNotNull(containerManager);
+ this.pipelineManager = pipelineManager;
+ this.containerManager = containerManager;
+ }
+
+ @Override
+ public void onMessage(
+ final IncrementalContainerReportFromDatanode containerReportFromDatanode,
+ final EventPublisher publisher) {
+
+ for (ContainerReplicaProto replicaProto :
+ containerReportFromDatanode.getReport().getReportList()) {
+ try {
+ final DatanodeDetails datanodeDetails = containerReportFromDatanode
+ .getDatanodeDetails();
+ final ContainerID containerID = ContainerID
+ .valueof(replicaProto.getContainerID());
+ final ContainerInfo containerInfo = containerManager
+ .getContainer(containerID);
+
+ ContainerReplica replica = ContainerReplica.newBuilder()
+ .setContainerID(ContainerID.valueof(replicaProto.getContainerID()))
+ .setContainerState(replicaProto.getState())
+ .setDatanodeDetails(datanodeDetails)
+ .build();
+
+ containerManager.updateContainerReplica(containerID, replica);
+
+ // Check if the state of the container is changed.
+ if (replicaProto.getState() == ContainerReplicaProto.State.CLOSED &&
+ containerInfo.getState() == HddsProtos.LifeCycleState.CLOSING) {
+ containerManager.updateContainerState(containerID,
+ HddsProtos.LifeCycleEvent.CLOSE);
+ }
+
+ // TODO: Handler replica state change
+
+ } catch (ContainerNotFoundException e) {
+ LOG.warn("Container {} not found!", replicaProto.getContainerID());
+ } catch (IOException e) {
+ LOG.error("Exception while processing ICR for container {}",
+ replicaProto.getContainerID());
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
index 0ea749f..86f1f9c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
@@ -23,13 +23,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
@@ -37,10 +34,6 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -489,106 +482,6 @@ public class SCMContainerManager implements ContainerManager {
}
/**
- * Process container report from Datanode.
- * <p>
- * Processing follows a very simple logic for time being.
- * <p>
- * 1. Datanodes report the current State -- denoted by the datanodeState
- * <p>
- * 2. We are the older SCM state from the Database -- denoted by
- * the knownState.
- * <p>
- * 3. We copy the usage etc. from currentState to newState and log that
- * newState to the DB. This allows us SCM to bootup again and read the
- * state of the world from the DB, and then reconcile the state from
- * container reports, when they arrive.
- *
- * @param reports Container report
- */
- @Override
- public void processContainerReports(DatanodeDetails datanodeDetails,
- ContainerReportsProto reports) throws IOException {
- List<ContainerReplicaProto>
- containerInfos = reports.getReportsList();
- PendingDeleteStatusList pendingDeleteStatusList =
- new PendingDeleteStatusList(datanodeDetails);
- for (ContainerReplicaProto newInfo :
- containerInfos) {
- ContainerID id = ContainerID.valueof(newInfo.getContainerID());
- ContainerReplica replica = ContainerReplica.newBuilder()
- .setContainerID(id)
- .setDatanodeDetails(datanodeDetails)
- .setOriginNodeId(datanodeDetails.getUuid())
- .build();
- lock.lock();
- try {
- containerStateManager.updateContainerReplica(id, replica);
- ContainerInfo currentInfo = containerStateManager.getContainer(id);
- if (newInfo.getState() == ContainerReplicaProto.State.CLOSED
- && currentInfo.getState() == LifeCycleState.CLOSING) {
- currentInfo = updateContainerStateInternal(id, LifeCycleEvent.CLOSE);
- if (!currentInfo.isOpen()) {
- pipelineManager.removeContainerFromPipeline(
- currentInfo.getPipelineID(), id);
- }
- }
-
- ContainerInfoProto newState =
- reconcileState(newInfo, currentInfo);
-
- if (currentInfo.getDeleteTransactionId() >
- newInfo.getDeleteTransactionId()) {
- pendingDeleteStatusList
- .addPendingDeleteStatus(newInfo.getDeleteTransactionId(),
- currentInfo.getDeleteTransactionId(),
- currentInfo.getContainerID());
- }
- containerStateManager.updateContainerInfo(
- ContainerInfo.fromProtobuf(newState));
- containerStore.put(id.getBytes(), newState.toByteArray());
- } catch (ContainerNotFoundException e) {
- LOG.error("Error while processing container report from datanode :" +
- " {}, for container: {}, reason: container doesn't exist in" +
- "container database.", datanodeDetails, id);
- } finally {
- lock.unlock();
- }
- }
- if (pendingDeleteStatusList.getNumPendingDeletes() > 0) {
- eventPublisher.fireEvent(SCMEvents.PENDING_DELETE_STATUS,
- pendingDeleteStatusList);
- }
-
- }
-
- /**
- * Reconciles the state from Datanode with the state in SCM.
- *
- * @param datanodeState - State from the Datanode.
- * @param knownState - State inside SCM.
- * @return new SCM State for this container.
- */
- private HddsProtos.ContainerInfoProto reconcileState(
- ContainerReplicaProto datanodeState,
- ContainerInfo knownState) {
- HddsProtos.ContainerInfoProto.Builder builder =
- HddsProtos.ContainerInfoProto.newBuilder();
- builder.setContainerID(knownState.getContainerID())
- .setPipelineID(knownState.getPipelineID().getProtobuf())
- .setState(knownState.getState())
- .setReplicationType(knownState.getReplicationType())
- .setReplicationFactor(knownState.getReplicationFactor())
- .setUsedBytes(datanodeState.getUsed())
- .setNumberOfKeys(datanodeState.getKeyCount())
- .setStateEnterTime(knownState.getStateEnterTime())
- .setDeleteTransactionId(knownState.getDeleteTransactionId());
- if (knownState.getOwner() != null) {
- builder.setOwner(knownState.getOwner());
- }
- return builder.build();
- }
-
- /**
* Returns the latest list of DataNodes where replica for given containerId
* exist. Throws an SCMException if no entry is found for given containerId.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
index b8052a4..0c738b2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdds.scm.container.states;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -263,12 +262,6 @@ public class ContainerStateMap {
}
}
- @VisibleForTesting
- // TODO: fix the test case and remove this method!
- public static Logger getLOG() {
- return LOG;
- }
-
/**
* Just update the container State.
* @param info ContainerInfo.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 30a7c34..72d416b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -21,15 +21,13 @@ package org.apache.hadoop.hdds.scm.events;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
-import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
- .CloseContainerStatus;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
.ReplicationStatus;
-import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler
- .CloseContainerRetryableReq;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .IncrementalContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.PipelineActionsFromDatanode;
@@ -79,6 +77,16 @@ public final class SCMEvents {
new TypedEvent<>(ContainerReportFromDatanode.class, "Container_Report");
/**
+ * IncrementalContainerReports are send out by Datanodes.
+ * This report is received by SCMDatanodeHeartbeatDispatcher and
+ * Incremental_Container_Report Event is generated.
+ */
+ public static final TypedEvent<IncrementalContainerReportFromDatanode>
+ INCREMENTAL_CONTAINER_REPORT = new TypedEvent<>(
+ IncrementalContainerReportFromDatanode.class,
+ "Incremental_Container_Report");
+
+ /**
* ContainerActions are sent by Datanode. This event is received by
* SCMDatanodeHeartbeatDispatcher and CONTAINER_ACTIONS event is generated.
*/
@@ -138,16 +146,6 @@ public final class SCMEvents {
new TypedEvent<>(ContainerID.class, "Close_Container");
/**
- * A CLOSE_CONTAINER_RETRYABLE_REQ will be triggered by
- * CloseContainerEventHandler after sending a SCMCommand to DataNode.
- * CloseContainerWatcher will track this event. Watcher will be responsible
- * for retrying it in event of failure or timeout.
- */
- public static final TypedEvent<CloseContainerRetryableReq>
- CLOSE_CONTAINER_RETRYABLE_REQ = new TypedEvent<>(
- CloseContainerRetryableReq.class, "Close_Container_Retryable");
-
- /**
* This event will be triggered whenever a new datanode is registered with
* SCM.
*/
@@ -176,14 +174,6 @@ public final class SCMEvents {
TypedEvent<>(ReplicationStatus.class, "Replicate_Command_Status");
/**
* This event will be triggered by CommandStatusReportHandler whenever a
- * status for CloseContainer SCMCommand is received.
- */
- public static final Event<CloseContainerStatus>
- CLOSE_CONTAINER_STATUS =
- new TypedEvent<>(CloseContainerStatus.class,
- "Close_Container_Command_Status");
- /**
- * This event will be triggered by CommandStatusReportHandler whenever a
* status for DeleteBlock SCMCommand is received.
*/
public static final TypedEvent<CommandStatusReportHandler.DeleteBlockStatus>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
index 1030428..43f0167 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -61,7 +62,14 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
// TODO: check if there are any pipeline on this node and fire close
// pipeline event
Set<ContainerID> ids =
- nodeManager.getContainers(datanodeDetails.getUuid());
+ null;
+ try {
+ ids = nodeManager.getContainers(datanodeDetails);
+ } catch (NodeNotFoundException e) {
+ // This should not happen, we cannot get a dead node event for an
+ // unregistered node!
+ LOG.error("DeadNode event for a unregistered node: {}!", datanodeDetails);
+ }
if (ids == null) {
LOG.info("There's no containers in dead datanode {}, no replica will be"
+ " removed from the in-memory state.", datanodeDetails.getUuid());
@@ -72,18 +80,23 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
for (ContainerID id : ids) {
try {
final ContainerInfo container = containerManager.getContainer(id);
+ // TODO: For open containers, trigger close on other nodes
+ // TODO: Check replica count and call replication manager
+ // on these containers.
if (!container.isOpen()) {
- final ContainerReplica replica = ContainerReplica.newBuilder()
- .setContainerID(id)
- .setDatanodeDetails(datanodeDetails)
- .build();
- try {
- containerManager.removeContainerReplica(id, replica);
- replicateIfNeeded(container, publisher);
- } catch (ContainerException ex) {
- LOG.warn("Exception while removing container replica #{} for " +
- "container #{}.", replica, container, ex);
- }
+ Set<ContainerReplica> replicas = containerManager
+ .getContainerReplicas(id);
+ replicas.stream()
+ .filter(r -> r.getDatanodeDetails().equals(datanodeDetails))
+ .findFirst()
+ .ifPresent(replica -> {
+ try {
+ containerManager.removeContainerReplica(id, replica);
+ } catch (ContainerException ex) {
+ LOG.warn("Exception while removing container replica #{} " +
+ "for container #{}.", replica, container, ex);
+ }
+ });
}
} catch (ContainerNotFoundException cnfe) {
LOG.warn("Container Not found!", cnfe);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
index 780aa2b..ed4fdba 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
@@ -19,31 +19,17 @@
package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
-import java.util.Collections;
-
/**
* Handles New Node event.
*/
public class NewNodeHandler implements EventHandler<DatanodeDetails> {
- private final NodeManager nodeManager;
-
- public NewNodeHandler(NodeManager nodeManager) {
- this.nodeManager = nodeManager;
- }
-
@Override
public void onMessage(DatanodeDetails datanodeDetails,
EventPublisher publisher) {
- try {
- nodeManager.addDatanodeInContainerMap(datanodeDetails.getUuid(),
- Collections.emptySet());
- } catch (SCMException e) {
- // TODO: log exception message.
- }
+ // We currently have nothing to do when we receive new node event.
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index d55ff98..e944634 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -63,13 +62,6 @@ import java.util.UUID;
*/
public interface NodeManager extends StorageContainerNodeProtocol,
EventHandler<CommandForDatanode>, NodeManagerMXBean, Closeable {
- /**
- * Removes a data node from the management of this Node Manager.
- *
- * @param node - DataNode.
- * @throws NodeNotFoundException
- */
- void removeNode(DatanodeDetails node) throws NodeNotFoundException;
/**
* Gets all Live Datanodes that is currently communicating with SCM.
@@ -102,6 +94,7 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* Return a map of node stats.
* @return a map of individual node stats (live/stale but not dead).
*/
+ // TODO: try to change the return type to Map<DatanodeDetails, SCMNodeStat>
Map<UUID, SCMNodeStat> getNodeStats();
/**
@@ -121,10 +114,10 @@ public interface NodeManager extends StorageContainerNodeProtocol,
/**
* Get set of pipelines a datanode is part of.
- * @param dnId - datanodeID
+ * @param datanodeDetails DatanodeDetails
* @return Set of PipelineID
*/
- Set<PipelineID> getPipelineByDnID(UUID dnId);
+ Set<PipelineID> getPipelines(DatanodeDetails datanodeDetails);
/**
* Add pipeline information in the NodeManager.
@@ -139,40 +132,22 @@ public interface NodeManager extends StorageContainerNodeProtocol,
void removePipeline(Pipeline pipeline);
/**
- * Update set of containers available on a datanode.
- * @param uuid - DatanodeID
+ * Remaps datanode to containers mapping to the new set of containers.
+ * @param datanodeDetails - DatanodeDetails
* @param containerIds - Set of containerIDs
* @throws SCMException - if datanode is not known. For new datanode use
* addDatanodeInContainerMap call.
*/
- void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
- throws SCMException;
-
- /**
- * Process containerReport received from datanode.
- * @param uuid - DataonodeID
- * @param containerIds - Set of containerIDs
- * @return The result after processing containerReport
- */
- ReportResult<ContainerID> processContainerReport(UUID uuid,
- Set<ContainerID> containerIds);
+ void setContainers(DatanodeDetails datanodeDetails,
+ Set<ContainerID> containerIds) throws NodeNotFoundException;
/**
* Return set of containerIDs available on a datanode.
- * @param uuid - DatanodeID
- * @return - set of containerIDs
- */
- Set<ContainerID> getContainers(UUID uuid);
-
- /**
- * Insert a new datanode with set of containerIDs for containers available
- * on it.
- * @param uuid - DatanodeID
- * @param containerIDs - Set of ContainerIDs
- * @throws SCMException - if datanode already exists
+ * @param datanodeDetails DatanodeDetails
+ * @return set of containerIDs
*/
- void addDatanodeInContainerMap(UUID uuid, Set<ContainerID> containerIDs)
- throws SCMException;
+ Set<ContainerID> getContainers(DatanodeDetails datanodeDetails)
+ throws NodeNotFoundException;
/**
* Add a {@link SCMCommand} to the command queue, which are
@@ -188,7 +163,7 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* @param dnUuid
* @param nodeReport
*/
- void processNodeReport(UUID dnUuid, NodeReportProto nodeReport);
+ void processNodeReport(DatanodeDetails dnUuid, NodeReportProto nodeReport);
/**
* Process a dead node event in this Node Manager.
@@ -202,5 +177,6 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* @param dnID - Datanode uuid.
* @return list of commands
*/
+ // TODO: We can give better name to this method!
List<SCMCommand> getCommandQueue(UUID dnID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
index 331bfed..f419764 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeReportHandler.java
@@ -50,6 +50,6 @@ public class NodeReportHandler implements EventHandler<NodeReportFromDatanode> {
+ "missing DatanodeDetails.");
LOGGER.trace("Processing node report for dn: {}", dn);
nodeManager
- .processNodeReport(dn.getUuid(), nodeReportFromDatanode.getReport());
+ .processNodeReport(dn, nodeReportFromDatanode.getReport());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index 588756c..a459519 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.states.*;
import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
import org.apache.hadoop.hdds.server.events.Event;
@@ -94,11 +93,6 @@ public class NodeStateManager implements Runnable, Closeable {
*/
private final Node2PipelineMap node2PipelineMap;
/**
- * Maintains the map from node to ContainerIDs for the containers
- * available on the node.
- */
- private final Node2ContainerMap node2ContainerMap;
- /**
* Used for publishing node state change events.
*/
private final EventPublisher eventPublisher;
@@ -131,7 +125,6 @@ public class NodeStateManager implements Runnable, Closeable {
public NodeStateManager(Configuration conf, EventPublisher eventPublisher) {
this.nodeStateMap = new NodeStateMap();
this.node2PipelineMap = new Node2PipelineMap();
- this.node2ContainerMap = new Node2ContainerMap();
this.eventPublisher = eventPublisher;
this.state2EventMap = new HashMap<>();
initialiseState2EventMap();
@@ -431,18 +424,6 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
- * Removes a node from NodeStateManager.
- *
- * @param datanodeDetails DatanodeDetails
- *
- * @throws NodeNotFoundException if the node is not present
- */
- public void removeNode(DatanodeDetails datanodeDetails)
- throws NodeNotFoundException {
- nodeStateMap.removeNode(datanodeDetails.getUuid());
- }
-
- /**
* Returns the current stats of the node.
*
* @param uuid node id
@@ -475,19 +456,6 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
- * Remove the current stats of the specify node.
- *
- * @param uuid node id
- *
- * @return SCMNodeStat the stat removed from the node.
- *
- * @throws NodeNotFoundException if the node is not present.
- */
- public SCMNodeStat removeNodeStat(UUID uuid) throws NodeNotFoundException {
- return nodeStateMap.removeNodeStat(uuid);
- }
-
- /**
* Removes a pipeline from the node2PipelineMap.
* @param pipeline - Pipeline to be removed
*/
@@ -498,23 +466,11 @@ public class NodeStateManager implements Runnable, Closeable {
* Update set of containers available on a datanode.
* @param uuid - DatanodeID
* @param containerIds - Set of containerIDs
- * @throws SCMException - if datanode is not known. For new datanode use
- * addDatanodeInContainerMap call.
- */
- public void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
- throws SCMException {
- node2ContainerMap.setContainersForDatanode(uuid, containerIds);
- }
-
- /**
- * Process containerReport received from datanode.
- * @param uuid - DataonodeID
- * @param containerIds - Set of containerIDs
- * @return The result after processing containerReport
+ * @throws NodeNotFoundException - if datanode is not known.
*/
- public ReportResult<ContainerID> processContainerReport(UUID uuid,
- Set<ContainerID> containerIds) {
- return node2ContainerMap.processReport(uuid, containerIds);
+ public void setContainers(UUID uuid, Set<ContainerID> containerIds)
+ throws NodeNotFoundException {
+ nodeStateMap.setContainers(uuid, containerIds);
}
/**
@@ -522,20 +478,9 @@ public class NodeStateManager implements Runnable, Closeable {
* @param uuid - DatanodeID
* @return - set of containerIDs
*/
- public Set<ContainerID> getContainers(UUID uuid) {
- return node2ContainerMap.getContainers(uuid);
- }
-
- /**
- * Insert a new datanode with set of containerIDs for containers available
- * on it.
- * @param uuid - DatanodeID
- * @param containerIDs - Set of ContainerIDs
- * @throws SCMException - if datanode already exists
- */
- public void addDatanodeInContainerMap(UUID uuid,
- Set<ContainerID> containerIDs) throws SCMException {
- node2ContainerMap.insertNewDatanode(uuid, containerIDs);
+ public Set<ContainerID> getContainers(UUID uuid)
+ throws NodeNotFoundException {
+ return nodeStateMap.getContainers(uuid);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 35c22f3..c42ef66 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
-import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.VersionInfo;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -129,16 +128,6 @@ public class SCMNodeManager
}
}
- /**
- * Removes a data node from the management of this Node Manager.
- *
- * @param node - DataNode.
- * @throws NodeNotFoundException
- */
- @Override
- public void removeNode(DatanodeDetails node) throws NodeNotFoundException {
- nodeStateManager.removeNode(node);
- }
/**
* Gets all datanodes that are in a certain state. This function works by
@@ -270,10 +259,8 @@ public class SCMNodeManager
datanodeDetails.setHostName(dnAddress.getHostName());
datanodeDetails.setIpAddress(dnAddress.getHostAddress());
}
- UUID dnId = datanodeDetails.getUuid();
try {
nodeStateManager.addNode(datanodeDetails);
- nodeStateManager.setNodeStat(dnId, new SCMNodeStat());
// Updating Node Report, as registration is successful
updateNodeStat(datanodeDetails.getUuid(), nodeReport);
LOG.info("Registered Data node : {}", datanodeDetails);
@@ -326,8 +313,9 @@ public class SCMNodeManager
* @param nodeReport
*/
@Override
- public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) {
- this.updateNodeStat(dnUuid, nodeReport);
+ public void processNodeReport(DatanodeDetails dnUuid,
+ NodeReportProto nodeReport) {
+ this.updateNodeStat(dnUuid.getUuid(), nodeReport);
}
/**
@@ -377,12 +365,12 @@ public class SCMNodeManager
/**
* Get set of pipelines a datanode is part of.
- * @param dnId - datanodeID
+ * @param datanodeDetails - datanodeID
* @return Set of PipelineID
*/
@Override
- public Set<PipelineID> getPipelineByDnID(UUID dnId) {
- return nodeStateManager.getPipelineByDnID(dnId);
+ public Set<PipelineID> getPipelines(DatanodeDetails datanodeDetails) {
+ return nodeStateManager.getPipelineByDnID(datanodeDetails.getUuid());
}
@@ -406,50 +394,27 @@ public class SCMNodeManager
/**
* Update set of containers available on a datanode.
- * @param uuid - DatanodeID
+ * @param datanodeDetails - DatanodeID
* @param containerIds - Set of containerIDs
* @throws SCMException - if datanode is not known. For new datanode use
* addDatanodeInContainerMap call.
*/
@Override
- public void setContainersForDatanode(UUID uuid,
- Set<ContainerID> containerIds) throws SCMException {
- nodeStateManager.setContainersForDatanode(uuid, containerIds);
- }
-
- /**
- * Process containerReport received from datanode.
- * @param uuid - DataonodeID
- * @param containerIds - Set of containerIDs
- * @return The result after processing containerReport
- */
- @Override
- public ReportResult<ContainerID> processContainerReport(UUID uuid,
- Set<ContainerID> containerIds) {
- return nodeStateManager.processContainerReport(uuid, containerIds);
+ public void setContainers(DatanodeDetails datanodeDetails,
+ Set<ContainerID> containerIds) throws NodeNotFoundException {
+ nodeStateManager.setContainers(datanodeDetails.getUuid(),
+ containerIds);
}
/**
* Return set of containerIDs available on a datanode.
- * @param uuid - DatanodeID
+ * @param datanodeDetails - DatanodeID
* @return - set of containerIDs
*/
@Override
- public Set<ContainerID> getContainers(UUID uuid) {
- return nodeStateManager.getContainers(uuid);
- }
-
- /**
- * Insert a new datanode with set of containerIDs for containers available
- * on it.
- * @param uuid - DatanodeID
- * @param containerIDs - Set of ContainerIDs
- * @throws SCMException - if datanode already exists
- */
- @Override
- public void addDatanodeInContainerMap(UUID uuid,
- Set<ContainerID> containerIDs) throws SCMException {
- nodeStateManager.addDatanodeInContainerMap(uuid, containerIDs);
+ public Set<ContainerID> getContainers(DatanodeDetails datanodeDetails)
+ throws NodeNotFoundException {
+ return nodeStateManager.getContainers(datanodeDetails.getUuid());
}
// TODO:
@@ -481,6 +446,7 @@ public class SCMNodeManager
* @param dnUuid datanode uuid.
*/
@Override
+ // TODO: This should be removed.
public void processDeadNode(UUID dnUuid) {
try {
SCMNodeStat stat = nodeStateManager.getNodeStat(dnUuid);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
index 9df9dff..268fe5b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
@@ -49,7 +49,7 @@ public class StaleNodeHandler implements EventHandler<DatanodeDetails> {
public void onMessage(DatanodeDetails datanodeDetails,
EventPublisher publisher) {
Set<PipelineID> pipelineIds =
- nodeManager.getPipelineByDnID(datanodeDetails.getUuid());
+ nodeManager.getPipelines(datanodeDetails);
for (PipelineID pipelineID : pipelineIds) {
try {
pipelineManager.finalizePipeline(pipelineID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
index 774ced1..a917e79 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.scm.node.states;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
@@ -48,6 +49,10 @@ public class NodeStateMap {
* Represents the current stats of node.
*/
private final ConcurrentHashMap<UUID, SCMNodeStat> nodeStats;
+ /**
+ * Node to set of containers on the node.
+ */
+ private final ConcurrentHashMap<UUID, Set<ContainerID>> nodeToContainer;
private final ReadWriteLock lock;
@@ -59,6 +64,7 @@ public class NodeStateMap {
nodeMap = new ConcurrentHashMap<>();
stateMap = new ConcurrentHashMap<>();
nodeStats = new ConcurrentHashMap<>();
+ nodeToContainer = new ConcurrentHashMap<>();
initStateMap();
}
@@ -88,6 +94,8 @@ public class NodeStateMap {
throw new NodeAlreadyExistsException("Node UUID: " + id);
}
nodeMap.put(id, new DatanodeInfo(datanodeDetails));
+ nodeStats.put(id, new SCMNodeStat());
+ nodeToContainer.put(id, Collections.emptySet());
stateMap.get(nodeState).add(id);
} finally {
lock.writeLock().unlock();
@@ -238,30 +246,6 @@ public class NodeStateMap {
}
/**
- * Removes the node from NodeStateMap.
- *
- * @param uuid node id
- *
- * @throws NodeNotFoundException if the node is not found
- */
- public void removeNode(UUID uuid) throws NodeNotFoundException {
- lock.writeLock().lock();
- try {
- if (nodeMap.containsKey(uuid)) {
- for (Map.Entry<NodeState, Set<UUID>> entry : stateMap.entrySet()) {
- if(entry.getValue().remove(uuid)) {
- break;
- }
- nodeMap.remove(uuid);
- }
- throw new NodeNotFoundException("Node UUID: " + uuid);
- }
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
* Returns the current stats of the node.
*
* @param uuid node id
@@ -298,21 +282,30 @@ public class NodeStateMap {
nodeStats.put(uuid, newstat);
}
- /**
- * Remove the current stats of the specify node.
- *
- * @param uuid node id
- *
- * @return SCMNodeStat the stat removed from the node.
- *
- * @throws NodeNotFoundException if the node is not found
- */
- public SCMNodeStat removeNodeStat(UUID uuid) throws NodeNotFoundException {
- SCMNodeStat stat = nodeStats.remove(uuid);
- if (stat == null) {
+ public void setContainers(UUID uuid, Set<ContainerID> containers)
+ throws NodeNotFoundException{
+ if (!nodeToContainer.containsKey(uuid)) {
throw new NodeNotFoundException("Node UUID: " + uuid);
}
- return stat;
+ nodeToContainer.put(uuid, containers);
+ }
+
+ public Set<ContainerID> getContainers(UUID uuid)
+ throws NodeNotFoundException {
+ Set<ContainerID> containers = nodeToContainer.get(uuid);
+ if (containers == null) {
+ throw new NodeNotFoundException("Node UUID: " + uuid);
+ }
+ return Collections.unmodifiableSet(containers);
+ }
+
+ public void removeContainer(UUID uuid, ContainerID containerID) throws
+ NodeNotFoundException {
+ Set<ContainerID> containers = nodeToContainer.get(uuid);
+ if (containers == null) {
+ throw new NodeNotFoundException("Node UUID: " + uuid);
+ }
+ containers.remove(containerID);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index 7c7df27..ede8b4f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.hdds.scm.server;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+ .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineActionsProto;
import org.apache.hadoop.hdds.protocol.proto
@@ -210,6 +212,19 @@ public final class SCMDatanodeHeartbeatDispatcher {
}
/**
+ * Incremental Container report event payload with origin.
+ */
+ public static class IncrementalContainerReportFromDatanode
+ extends ReportFromDatanode<IncrementalContainerReportProto> {
+
+ public IncrementalContainerReportFromDatanode(
+ DatanodeDetails datanodeDetails,
+ IncrementalContainerReportProto report) {
+ super(datanodeDetails, report);
+ }
+ }
+
+ /**
* Container action event payload with origin.
*/
public static class ContainerActionsFromDatanode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 37c7386..0beceab 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -97,6 +97,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRES
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT;
import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
@@ -196,8 +197,9 @@ public class SCMDatanodeProtocolServer implements
.register(datanodeDetails, nodeReport, pipelineReportsProto);
if (registeredCommand.getError()
== SCMRegisteredResponseProto.ErrorCode.success) {
- scm.getContainerManager().processContainerReports(datanodeDetails,
- containerReportsProto);
+ eventPublisher.fireEvent(CONTAINER_REPORT,
+ new SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode(
+ datanodeDetails, containerReportsProto));
eventPublisher.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
new NodeRegistrationContainerReport(datanodeDetails,
containerReportsProto));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/29] hadoop git commit: HDDS-811. [JDK 11] Fix compilation
failures with jdk 11. Contributed by Lokesh Jain.
Posted by ae...@apache.org.
HDDS-811. [JDK 11] Fix compilation failures with jdk 11. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49412a12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49412a12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49412a12
Branch: refs/heads/HDDS-4
Commit: 49412a12898eda60cb460ebf4bb9126f3c8c1f53
Parents: 78e43bc
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 6 21:26:43 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 6 21:26:43 2018 +0900
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 6 ++++++
hadoop-ozone/s3gateway/pom.xml | 4 ++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49412a12/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 65cd1d1..0345012 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -37,6 +37,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependencies>
<dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>1.2</version>
+ </dependency>
+
+ <dependency>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49412a12/hadoop-ozone/s3gateway/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index 5bc7c62..b78796f 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -38,6 +38,10 @@
<dependencies>
<dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
<scope>compile</scope>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/29] hadoop git commit: HDDS-809. Refactor SCMChillModeManager.
Posted by ae...@apache.org.
HDDS-809. Refactor SCMChillModeManager.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/addec292
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/addec292
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/addec292
Branch: refs/heads/HDDS-4
Commit: addec29297e61a417f0ce711bd76b6db53d504eb
Parents: 482716e
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Nov 7 13:53:28 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Wed Nov 7 13:54:08 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/scm/ScmUtils.java | 2 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 2 +-
.../hdds/scm/chillmode/ChillModeExitRule.java | 32 ++
.../hdds/scm/chillmode/ChillModePrecheck.java | 68 ++++
.../scm/chillmode/ChillModeRestrictedOps.java | 41 +++
.../scm/chillmode/ContainerChillModeRule.java | 112 +++++++
.../scm/chillmode/DataNodeChillModeRule.java | 83 +++++
.../hadoop/hdds/scm/chillmode/Precheck.java | 29 ++
.../hdds/scm/chillmode/SCMChillModeManager.java | 153 +++++++++
.../hadoop/hdds/scm/chillmode/package-info.java | 18 ++
.../hdds/scm/server/ChillModePrecheck.java | 69 ----
.../apache/hadoop/hdds/scm/server/Precheck.java | 29 --
.../hdds/scm/server/SCMChillModeManager.java | 319 -------------------
.../scm/server/SCMClientProtocolServer.java | 1 +
.../scm/server/StorageContainerManager.java | 1 +
.../scm/chillmode/TestSCMChillModeManager.java | 215 +++++++++++++
.../scm/server/TestSCMChillModeManager.java | 215 -------------
.../hadoop/ozone/om/TestScmChillMode.java | 2 +-
18 files changed, 756 insertions(+), 635 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
index 435f0a5..43b4452 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+import org.apache.hadoop.hdds.scm.chillmode.Precheck;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.server.Precheck;
/**
* SCM utility class.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index c878d97..85658b9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmUtils;
+import org.apache.hadoop.hdds.scm.chillmode.ChillModePrecheck;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -32,7 +33,6 @@ import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.server.ChillModePrecheck;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.metrics2.util.MBeans;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java
new file mode 100644
index 0000000..d283dfe
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeExitRule.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+/**
+ * Interface for defining chill mode exit rules.
+ *
+ * @param <T>
+ */
+public interface ChillModeExitRule<T> {
+
+ boolean validate();
+
+ void process(T report);
+
+ void cleanup();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java
new file mode 100644
index 0000000..0ed06dd
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModePrecheck.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
+
+/**
+ * Chill mode pre-check for SCM operations.
+ * */
+public class ChillModePrecheck implements Precheck<ScmOps> {
+
+ private AtomicBoolean inChillMode;
+ public static final String PRECHECK_TYPE = "ChillModePrecheck";
+
+ public ChillModePrecheck(Configuration conf) {
+ boolean chillModeEnabled = conf.getBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
+ if (chillModeEnabled) {
+ inChillMode = new AtomicBoolean(true);
+ } else {
+ inChillMode = new AtomicBoolean(false);
+ }
+ }
+
+ public boolean check(ScmOps op) throws SCMException {
+ if (inChillMode.get() && ChillModeRestrictedOps
+ .isRestrictedInChillMode(op)) {
+ throw new SCMException("ChillModePrecheck failed for " + op,
+ ResultCodes.CHILL_MODE_EXCEPTION);
+ }
+ return inChillMode.get();
+ }
+
+ @Override
+ public String type() {
+ return PRECHECK_TYPE;
+ }
+
+ public boolean isInChillMode() {
+ return inChillMode.get();
+ }
+
+ public void setInChillMode(boolean inChillMode) {
+ this.inChillMode.set(inChillMode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java
new file mode 100644
index 0000000..f3b003b
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ChillModeRestrictedOps.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.EnumSet;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+
+/**
+ * Operations restricted in SCM chill mode.
+ */
+public final class ChillModeRestrictedOps {
+ private static EnumSet restrictedOps = EnumSet.noneOf(ScmOps.class);
+
+ private ChillModeRestrictedOps() {
+ }
+
+ static {
+ restrictedOps.add(ScmOps.allocateBlock);
+ restrictedOps.add(ScmOps.allocateContainer);
+ }
+
+ public static boolean isRestrictedInChillMode(ScmOps opName) {
+ return restrictedOps.contains(opName);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
new file mode 100644
index 0000000..57eb8dd
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/ContainerChillModeRule.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Class defining Chill mode exit criteria for Containers.
+ */
+public class ContainerChillModeRule implements
+ ChillModeExitRule<NodeRegistrationContainerReport> {
+
+ // Required cutoff % for containers with at least 1 reported replica.
+ private double chillModeCutoff;
+ // Containers read from scm db (excluding containers in ALLOCATED state).
+ private Map<Long, ContainerInfo> containerMap;
+ private double maxContainer;
+
+ private AtomicLong containerWithMinReplicas = new AtomicLong(0);
+ private final SCMChillModeManager chillModeManager;
+
+ public ContainerChillModeRule(Configuration conf,
+ List<ContainerInfo> containers, SCMChillModeManager manager) {
+ chillModeCutoff = conf.getDouble(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT,
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT);
+ chillModeManager = manager;
+ containerMap = new ConcurrentHashMap<>();
+ if(containers != null) {
+ containers.forEach(c -> {
+ // Containers in ALLOCATED state should not be included while
+ // calculating the total number of containers here. They are not
+ // reported by DNs and hence should not affect the chill mode exit
+ // rule.
+ if (c != null && c.getState() != null &&
+ !c.getState().equals(HddsProtos.LifeCycleState.ALLOCATED)) {
+ containerMap.put(c.getContainerID(), c);
+ }
+ });
+ maxContainer = containerMap.size();
+ }
+ }
+
+ @Override
+ public boolean validate() {
+ if (maxContainer == 0) {
+ return true;
+ }
+ return getCurrentContainerThreshold() >= chillModeCutoff;
+ }
+
+ @VisibleForTesting
+ public double getCurrentContainerThreshold() {
+ if (maxContainer == 0) {
+ return 1;
+ }
+ return (containerWithMinReplicas.doubleValue() / maxContainer);
+ }
+
+ @Override
+ public void process(NodeRegistrationContainerReport reportsProto) {
+ if (maxContainer == 0) {
+ // No container to check.
+ return;
+ }
+
+ reportsProto.getReport().getReportsList().forEach(c -> {
+ if (containerMap.containsKey(c.getContainerID())) {
+ if(containerMap.remove(c.getContainerID()) != null) {
+ containerWithMinReplicas.getAndAdd(1);
+ }
+ }
+ });
+ if(chillModeManager.getInChillMode()) {
+ SCMChillModeManager.getLogger().info(
+ "SCM in chill mode. {} % containers have at least one"
+ + " reported replica.",
+ (containerWithMinReplicas.get() / maxContainer) * 100);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ containerMap.clear();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
new file mode 100644
index 0000000..3950505
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/DataNodeChillModeRule.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.HashSet;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Class defining Chill mode exit criteria according to number of DataNodes
+ * registered with SCM.
+ */
+public class DataNodeChillModeRule implements
+ ChillModeExitRule<NodeRegistrationContainerReport> {
+
+ // Min DataNodes required to exit chill mode.
+ private int requiredDns;
+ private int registeredDns = 0;
+ // Set to track registered DataNodes.
+ private HashSet<UUID> registeredDnSet;
+
+ private final SCMChillModeManager chillModeManager;
+
+ public DataNodeChillModeRule(Configuration conf,
+ SCMChillModeManager manager) {
+ requiredDns = conf.getInt(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE,
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE_DEFAULT);
+ registeredDnSet = new HashSet<>(requiredDns * 2);
+ chillModeManager = manager;
+ }
+
+ @Override
+ public boolean validate() {
+ return registeredDns >= requiredDns;
+ }
+
+ @VisibleForTesting
+ public double getRegisteredDataNodes() {
+ return registeredDns;
+ }
+
+ @Override
+ public void process(NodeRegistrationContainerReport reportsProto) {
+ if (requiredDns == 0) {
+ // No dn check required.
+ return;
+ }
+
+ if(chillModeManager.getInChillMode()) {
+ registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
+ registeredDns = registeredDnSet.size();
+ SCMChillModeManager.getLogger().info(
+ "SCM in chill mode. {} DataNodes registered, {} required.",
+ registeredDns, requiredDns);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ registeredDnSet.clear();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java
new file mode 100644
index 0000000..30daa85
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/Precheck.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+
+/**
+ * Precheck for SCM operations.
+ * */
+public interface Precheck<T> {
+ boolean check(T t) throws SCMException;
+ String type();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
new file mode 100644
index 0000000..fa0eaa9
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
+ .NodeRegistrationContainerReport;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * StorageContainerManager enters chill mode on startup to allow system to
+ * reach a stable state before becoming fully functional. SCM will wait
+ * for certain resources to be reported before coming out of chill mode.
+ *
+ * ChillModeExitRule defines format to define new rules which must be satisfied
+ * to exit Chill mode.
+ * ContainerChillModeRule defines the only exit criteria right now.
+ * On every new datanode registration event this class adds replicas
+ * for reported containers and validates if cutoff threshold for
+ * containers is meet.
+ */
+public class SCMChillModeManager implements
+ EventHandler<NodeRegistrationContainerReport> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SCMChillModeManager.class);
+ private final boolean isChillModeEnabled;
+ private AtomicBoolean inChillMode = new AtomicBoolean(true);
+ private Map<String, ChillModeExitRule> exitRules = new HashMap(1);
+ private Configuration config;
+ private static final String CONT_EXIT_RULE = "ContainerChillModeRule";
+ private static final String DN_EXIT_RULE = "DataNodeChillModeRule";
+ private final EventQueue eventPublisher;
+
+ public SCMChillModeManager(Configuration conf,
+ List<ContainerInfo> allContainers, EventQueue eventQueue) {
+ this.config = conf;
+ this.eventPublisher = eventQueue;
+ this.isChillModeEnabled = conf.getBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
+ if (isChillModeEnabled) {
+ exitRules.put(CONT_EXIT_RULE,
+ new ContainerChillModeRule(config, allContainers, this));
+ exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this));
+ emitChillModeStatus();
+ } else {
+ exitChillMode(eventQueue);
+ }
+ }
+
+ /**
+ * Emit Chill mode status.
+ */
+ @VisibleForTesting
+ public void emitChillModeStatus() {
+ eventPublisher.fireEvent(SCMEvents.CHILL_MODE_STATUS, getInChillMode());
+ }
+
+ private void validateChillModeExitRules(EventPublisher eventQueue) {
+ for (ChillModeExitRule exitRule : exitRules.values()) {
+ if (!exitRule.validate()) {
+ return;
+ }
+ }
+ exitChillMode(eventQueue);
+ }
+
+ /**
+ * Exit chill mode. It does following actions:
+ * 1. Set chill mode status to false.
+ * 2. Emits START_REPLICATION for ReplicationManager.
+ * 3. Cleanup resources.
+ * 4. Emit chill mode status.
+ * @param eventQueue
+ */
+ @VisibleForTesting
+ public void exitChillMode(EventPublisher eventQueue) {
+ LOG.info("SCM exiting chill mode.");
+ setInChillMode(false);
+
+ // TODO: Remove handler registration as there is no need to listen to
+ // register events anymore.
+
+ for (ChillModeExitRule e : exitRules.values()) {
+ e.cleanup();
+ }
+ emitChillModeStatus();
+ }
+
+ @Override
+ public void onMessage(
+ NodeRegistrationContainerReport nodeRegistrationContainerReport,
+ EventPublisher publisher) {
+ if (getInChillMode()) {
+ exitRules.get(CONT_EXIT_RULE).process(nodeRegistrationContainerReport);
+ exitRules.get(DN_EXIT_RULE).process(nodeRegistrationContainerReport);
+ validateChillModeExitRules(publisher);
+ }
+ }
+
+ public boolean getInChillMode() {
+ if (!isChillModeEnabled) {
+ return false;
+ }
+ return inChillMode.get();
+ }
+
+ /**
+ * Set chill mode status.
+ */
+ public void setInChillMode(boolean inChillMode) {
+ this.inChillMode.set(inChillMode);
+ }
+
+ public static Logger getLogger() {
+ return LOG;
+ }
+
+ @VisibleForTesting
+ public double getCurrentContainerThreshold() {
+ return ((ContainerChillModeRule) exitRules.get(CONT_EXIT_RULE))
+ .getCurrentContainerThreshold();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java
new file mode 100644
index 0000000..2513ac7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
deleted file mode 100644
index a3f4c90..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.server;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
-import org.apache.hadoop.hdds.scm.server.SCMChillModeManager.ChillModeRestrictedOps;
-
-/**
- * Chill mode pre-check for SCM operations.
- * */
-public class ChillModePrecheck implements Precheck<ScmOps> {
-
- private AtomicBoolean inChillMode;
- public static final String PRECHECK_TYPE = "ChillModePrecheck";
-
- public ChillModePrecheck(Configuration conf) {
- boolean chillModeEnabled = conf.getBoolean(
- HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
- HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
- if (chillModeEnabled) {
- inChillMode = new AtomicBoolean(true);
- } else {
- inChillMode = new AtomicBoolean(false);
- }
- }
-
- public boolean check(ScmOps op) throws SCMException {
- if (inChillMode.get() && ChillModeRestrictedOps
- .isRestrictedInChillMode(op)) {
- throw new SCMException("ChillModePrecheck failed for " + op,
- ResultCodes.CHILL_MODE_EXCEPTION);
- }
- return inChillMode.get();
- }
-
- @Override
- public String type() {
- return PRECHECK_TYPE;
- }
-
- public boolean isInChillMode() {
- return inChillMode.get();
- }
-
- public void setInChillMode(boolean inChillMode) {
- this.inChillMode.set(inChillMode);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java
deleted file mode 100644
index 1654990..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/Precheck.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.server;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-
-/**
- * Precheck for SCM operations.
- * */
-public interface Precheck<T> {
- boolean check(T t) throws SCMException;
- String type();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
deleted file mode 100644
index 6a342d4..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.server;
-
-import com.google.common.annotations.VisibleForTesting;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
- .NodeRegistrationContainerReport;
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * StorageContainerManager enters chill mode on startup to allow system to
- * reach a stable state before becoming fully functional. SCM will wait
- * for certain resources to be reported before coming out of chill mode.
- *
- * ChillModeExitRule defines format to define new rules which must be satisfied
- * to exit Chill mode.
- * ContainerChillModeRule defines the only exit criteria right now.
- * On every new datanode registration event this class adds replicas
- * for reported containers and validates if cutoff threshold for
- * containers is meet.
- */
-public class SCMChillModeManager implements
- EventHandler<NodeRegistrationContainerReport> {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMChillModeManager.class);
- private final boolean isChillModeEnabled;
- private AtomicBoolean inChillMode = new AtomicBoolean(true);
- private AtomicLong containerWithMinReplicas = new AtomicLong(0);
- private Map<String, ChillModeExitRule> exitRules = new HashMap(1);
- private Configuration config;
- private static final String CONT_EXIT_RULE = "ContainerChillModeRule";
- private static final String DN_EXIT_RULE = "DataNodeChillModeRule";
- private final EventQueue eventPublisher;
-
- SCMChillModeManager(Configuration conf, List<ContainerInfo> allContainers,
- EventQueue eventQueue) {
- this.config = conf;
- this.eventPublisher = eventQueue;
- this.isChillModeEnabled = conf.getBoolean(
- HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED,
- HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED_DEFAULT);
- if (isChillModeEnabled) {
- exitRules.put(CONT_EXIT_RULE,
- new ContainerChillModeRule(config, allContainers));
- exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config));
- emitChillModeStatus();
- } else {
- exitChillMode(eventQueue);
- }
- }
-
- /**
- * Emit Chill mode status.
- */
- @VisibleForTesting
- public void emitChillModeStatus() {
- eventPublisher.fireEvent(SCMEvents.CHILL_MODE_STATUS, getInChillMode());
- }
-
- private void validateChillModeExitRules(EventPublisher eventQueue) {
- for (ChillModeExitRule exitRule : exitRules.values()) {
- if (!exitRule.validate()) {
- return;
- }
- }
- exitChillMode(eventQueue);
- }
-
- /**
- * Exit chill mode. It does following actions:
- * 1. Set chill mode status to false.
- * 2. Emits START_REPLICATION for ReplicationManager.
- * 3. Cleanup resources.
- * 4. Emit chill mode status.
- * @param eventQueue
- */
- @VisibleForTesting
- public void exitChillMode(EventPublisher eventQueue) {
- LOG.info("SCM exiting chill mode.");
- setInChillMode(false);
-
- // TODO: Remove handler registration as there is no need to listen to
- // register events anymore.
-
- for (ChillModeExitRule e : exitRules.values()) {
- e.cleanup();
- }
- emitChillModeStatus();
- }
-
- @Override
- public void onMessage(
- NodeRegistrationContainerReport nodeRegistrationContainerReport,
- EventPublisher publisher) {
- if (getInChillMode()) {
- exitRules.get(CONT_EXIT_RULE).process(nodeRegistrationContainerReport);
- exitRules.get(DN_EXIT_RULE).process(nodeRegistrationContainerReport);
- validateChillModeExitRules(publisher);
- }
- }
-
- public boolean getInChillMode() {
- if (!isChillModeEnabled) {
- return false;
- }
- return inChillMode.get();
- }
-
- /**
- * Set chill mode status.
- */
- public void setInChillMode(boolean inChillMode) {
- this.inChillMode.set(inChillMode);
- }
-
- /**
- * Interface for defining chill mode exit rules.
- *
- * @param <T>
- */
- public interface ChillModeExitRule<T> {
-
- boolean validate();
-
- void process(T report);
-
- void cleanup();
- }
-
- /**
- * Class defining Chill mode exit criteria for Containers.
- */
- public class ContainerChillModeRule implements
- ChillModeExitRule<NodeRegistrationContainerReport> {
-
- // Required cutoff % for containers with at least 1 reported replica.
- private double chillModeCutoff;
- // Containers read from scm db (excluding containers in ALLOCATED state).
- private Map<Long, ContainerInfo> containerMap;
- private double maxContainer;
-
- public ContainerChillModeRule(Configuration conf,
- List<ContainerInfo> containers) {
- chillModeCutoff = conf
- .getDouble(HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT,
- HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT);
- containerMap = new ConcurrentHashMap<>();
- if(containers != null) {
- containers.forEach(c -> {
- // Containers in ALLOCATED state should not be included while
- // calculating the total number of containers here. They are not
- // reported by DNs and hence should not affect the chill mode exit
- // rule.
- if (c != null && c.getState() != null &&
- !c.getState().equals(HddsProtos.LifeCycleState.ALLOCATED)) {
- containerMap.put(c.getContainerID(), c);
- }
- });
- maxContainer = containerMap.size();
- }
- }
-
- @Override
- public boolean validate() {
- if (maxContainer == 0) {
- return true;
- }
- return getCurrentContainerThreshold() >= chillModeCutoff;
- }
-
- @VisibleForTesting
- public double getCurrentContainerThreshold() {
- if (maxContainer == 0) {
- return 1;
- }
- return (containerWithMinReplicas.doubleValue() / maxContainer);
- }
-
- @Override
- public void process(NodeRegistrationContainerReport reportsProto) {
- if (maxContainer == 0) {
- // No container to check.
- return;
- }
-
- reportsProto.getReport().getReportsList().forEach(c -> {
- if (containerMap.containsKey(c.getContainerID())) {
- if(containerMap.remove(c.getContainerID()) != null) {
- containerWithMinReplicas.getAndAdd(1);
- }
- }
- });
- if(getInChillMode()) {
- LOG.info("SCM in chill mode. {} % containers have at least one"
- + " reported replica.",
- (containerWithMinReplicas.get() / maxContainer) * 100);
- }
- }
-
- @Override
- public void cleanup() {
- containerMap.clear();
- }
- }
-
- /**
- * Class defining Chill mode exit criteria according to number of DataNodes
- * registered with SCM.
- */
- public class DataNodeChillModeRule implements
- ChillModeExitRule<NodeRegistrationContainerReport> {
-
- // Min DataNodes required to exit chill mode.
- private int requiredDns;
- private int registeredDns = 0;
- // Set to track registered DataNodes.
- private HashSet<UUID> registeredDnSet;
-
- public DataNodeChillModeRule(Configuration conf) {
- requiredDns = conf
- .getInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE,
- HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE_DEFAULT);
- registeredDnSet = new HashSet<>(requiredDns * 2);
- }
-
- @Override
- public boolean validate() {
- return registeredDns >= requiredDns;
- }
-
- @VisibleForTesting
- public double getRegisteredDataNodes() {
- return registeredDns;
- }
-
- @Override
- public void process(NodeRegistrationContainerReport reportsProto) {
- if (requiredDns == 0) {
- // No dn check required.
- return;
- }
-
- if(getInChillMode()) {
- registeredDnSet.add(reportsProto.getDatanodeDetails().getUuid());
- registeredDns = registeredDnSet.size();
- LOG.info("SCM in chill mode. {} DataNodes registered, {} required.",
- registeredDns, requiredDns);
- }
- }
-
- @Override
- public void cleanup() {
- registeredDnSet.clear();
- }
- }
-
- @VisibleForTesting
- public static Logger getLogger() {
- return LOG;
- }
-
- @VisibleForTesting
- public double getCurrentContainerThreshold() {
- return ((ContainerChillModeRule) exitRules.get(CONT_EXIT_RULE))
- .getCurrentContainerThreshold();
- }
-
- /**
- * Operations restricted in SCM chill mode.
- */
- public static class ChillModeRestrictedOps {
- private static EnumSet restrictedOps = EnumSet.noneOf(ScmOps.class);
-
- static {
- restrictedOps.add(ScmOps.allocateBlock);
- restrictedOps.add(ScmOps.allocateContainer);
- }
-
- public static boolean isRestrictedInChillMode(ScmOps opName) {
- return restrictedOps.contains(opName);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 58cb871..181bcc5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.ScmUtils;
+import org.apache.hadoop.hdds.scm.chillmode.ChillModePrecheck;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index d8b9958..c1409cb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
import org.apache.hadoop.hdds.scm.block.PendingDeleteHandler;
+import org.apache.hadoop.hdds.scm.chillmode.SCMChillModeManager;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
import org.apache.hadoop.hdds.scm.container.CloseContainerWatcher;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
new file mode 100644
index 0000000..2c08077
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.HddsTestUtils;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+/** Test class for SCMChillModeManager.
+ */
+public class TestSCMChillModeManager {
+
+ private static EventQueue queue;
+ private SCMChillModeManager scmChillModeManager;
+ private static Configuration config;
+ private List<ContainerInfo> containers;
+
+ @Rule
+ public Timeout timeout = new Timeout(1000 * 35);
+
+ @BeforeClass
+ public static void setUp() {
+ queue = new EventQueue();
+ config = new OzoneConfiguration();
+ }
+
+ @Test
+ public void testChillModeState() throws Exception {
+ // Test 1: test for 0 containers
+ testChillMode(0);
+
+ // Test 2: test for 20 containers
+ testChillMode(20);
+ }
+
+ @Test
+ public void testChillModeStateWithNullContainers() {
+ new SCMChillModeManager(config, null, queue);
+ }
+
+ private void testChillMode(int numContainers) throws Exception {
+ containers = new ArrayList<>();
+ containers.addAll(HddsTestUtils.getContainerInfo(numContainers));
+ // Assign open state to containers to be included in the chill mode
+ // container list
+ for (ContainerInfo container : containers) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
+ scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ assertTrue(scmChillModeManager.getInChillMode());
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(containers));
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 100, 1000 * 5);
+ }
+
+ @Test
+ public void testChillModeExitRule() throws Exception {
+ containers = new ArrayList<>();
+ containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ // Assign open state to containers to be included in the chill mode
+ // container list
+ for (ContainerInfo container : containers) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
+ scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ testContainerThreshold(containers.subList(0, 25), 0.25);
+ assertTrue(scmChillModeManager.getInChillMode());
+ testContainerThreshold(containers.subList(25, 50), 0.50);
+ assertTrue(scmChillModeManager.getInChillMode());
+ testContainerThreshold(containers.subList(50, 75), 0.75);
+ assertTrue(scmChillModeManager.getInChillMode());
+ testContainerThreshold(containers.subList(75, 100), 1.0);
+
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 100, 1000 * 5);
+ }
+
+ @Test
+ public void testDisableChillMode() {
+ OzoneConfiguration conf = new OzoneConfiguration(config);
+ conf.setBoolean(HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, false);
+ scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+ assertFalse(scmChillModeManager.getInChillMode());
+ }
+
+ @Test
+ public void testChillModeDataNodeExitRule() throws Exception {
+ containers = new ArrayList<>();
+ testChillModeDataNodes(0);
+ testChillModeDataNodes(3);
+ testChillModeDataNodes(5);
+ }
+
+ /**
+ * Check that containers in Allocated state are not considered while
+ * computing percentage of containers with at least 1 reported replica in
+ * chill mode exit rule.
+ */
+ @Test
+ public void testContainerChillModeRule() throws Exception {
+ containers = new ArrayList<>();
+ // Add 100 containers to the list of containers in SCM
+ containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ // Assign OPEN state to first 25 containers and ALLLOCATED state to rest
+ // of the containers
+ for (ContainerInfo container : containers.subList(0, 25)) {
+ container.setState(HddsProtos.LifeCycleState.OPEN);
+ }
+ for (ContainerInfo container : containers.subList(25, 100)) {
+ container.setState(HddsProtos.LifeCycleState.ALLOCATED);
+ }
+
+ scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // When 10 OPEN containers are reported by DNs, the computed container
+ // threshold should be 10/25 as there are only 25 open containers.
+ // Containers in ALLOCATED state should not contribute towards list of
+ // containers while calculating container threshold in SCMChillNodeManager
+ testContainerThreshold(containers.subList(0, 10), 0.4);
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // When remaining 15 OPEN containers are reported by DNs, the container
+ // threshold should be (10+15)/25.
+ testContainerThreshold(containers.subList(10, 25), 1.0);
+
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 100, 1000 * 5);
+ }
+
+ private void testChillModeDataNodes(int numOfDns) throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration(config);
+ conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
+ scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ // Assert SCM is in Chill mode.
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // Register all DataNodes except last one and assert SCM is in chill mode.
+ for (int i = 0; i < numOfDns-1; i++) {
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(containers));
+ assertTrue(scmChillModeManager.getInChillMode());
+ assertTrue(scmChillModeManager.getCurrentContainerThreshold() == 1);
+ }
+
+ if(numOfDns == 0){
+ GenericTestUtils.waitFor(() -> {
+ return scmChillModeManager.getInChillMode();
+ }, 10, 1000 * 10);
+ return;
+ }
+ // Register last DataNode and check that SCM is out of Chill mode.
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(containers));
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 10, 1000 * 10);
+ }
+
+ private void testContainerThreshold(List<ContainerInfo> dnContainers,
+ double expectedThreshold)
+ throws Exception {
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(dnContainers));
+ GenericTestUtils.waitFor(() -> {
+ double threshold = scmChillModeManager.getCurrentContainerThreshold();
+ return threshold == expectedThreshold;
+ }, 100, 2000 * 9);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
deleted file mode 100644
index ef28354..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMChillModeManager.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.server;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.HddsTestUtils;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
-/** Test class for SCMChillModeManager.
- */
-public class TestSCMChillModeManager {
-
- private static EventQueue queue;
- private SCMChillModeManager scmChillModeManager;
- private static Configuration config;
- private List<ContainerInfo> containers;
-
- @Rule
- public Timeout timeout = new Timeout(1000 * 35);
-
- @BeforeClass
- public static void setUp() {
- queue = new EventQueue();
- config = new OzoneConfiguration();
- }
-
- @Test
- public void testChillModeState() throws Exception {
- // Test 1: test for 0 containers
- testChillMode(0);
-
- // Test 2: test for 20 containers
- testChillMode(20);
- }
-
- @Test
- public void testChillModeStateWithNullContainers() {
- new SCMChillModeManager(config, null, queue);
- }
-
- private void testChillMode(int numContainers) throws Exception {
- containers = new ArrayList<>();
- containers.addAll(HddsTestUtils.getContainerInfo(numContainers));
- // Assign open state to containers to be included in the chill mode
- // container list
- for (ContainerInfo container : containers) {
- container.setState(HddsProtos.LifeCycleState.OPEN);
- }
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
- assertTrue(scmChillModeManager.getInChillMode());
- queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- HddsTestUtils.createNodeRegistrationContainerReport(containers));
- GenericTestUtils.waitFor(() -> {
- return !scmChillModeManager.getInChillMode();
- }, 100, 1000 * 5);
- }
-
- @Test
- public void testChillModeExitRule() throws Exception {
- containers = new ArrayList<>();
- containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
- // Assign open state to containers to be included in the chill mode
- // container list
- for (ContainerInfo container : containers) {
- container.setState(HddsProtos.LifeCycleState.OPEN);
- }
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
- assertTrue(scmChillModeManager.getInChillMode());
-
- testContainerThreshold(containers.subList(0, 25), 0.25);
- assertTrue(scmChillModeManager.getInChillMode());
- testContainerThreshold(containers.subList(25, 50), 0.50);
- assertTrue(scmChillModeManager.getInChillMode());
- testContainerThreshold(containers.subList(50, 75), 0.75);
- assertTrue(scmChillModeManager.getInChillMode());
- testContainerThreshold(containers.subList(75, 100), 1.0);
-
- GenericTestUtils.waitFor(() -> {
- return !scmChillModeManager.getInChillMode();
- }, 100, 1000 * 5);
- }
-
- @Test
- public void testDisableChillMode() {
- OzoneConfiguration conf = new OzoneConfiguration(config);
- conf.setBoolean(HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, false);
- scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
- assertFalse(scmChillModeManager.getInChillMode());
- }
-
- @Test
- public void testChillModeDataNodeExitRule() throws Exception {
- containers = new ArrayList<>();
- testChillModeDataNodes(0);
- testChillModeDataNodes(3);
- testChillModeDataNodes(5);
- }
-
- /**
- * Check that containers in Allocated state are not considered while
- * computing percentage of containers with at least 1 reported replica in
- * chill mode exit rule.
- */
- @Test
- public void testContainerChillModeRule() throws Exception {
- containers = new ArrayList<>();
- // Add 100 containers to the list of containers in SCM
- containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
- // Assign OPEN state to first 25 containers and ALLLOCATED state to rest
- // of the containers
- for (ContainerInfo container : containers.subList(0, 25)) {
- container.setState(HddsProtos.LifeCycleState.OPEN);
- }
- for (ContainerInfo container : containers.subList(25, 100)) {
- container.setState(HddsProtos.LifeCycleState.ALLOCATED);
- }
-
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
- assertTrue(scmChillModeManager.getInChillMode());
-
- // When 10 OPEN containers are reported by DNs, the computed container
- // threshold should be 10/25 as there are only 25 open containers.
- // Containers in ALLOCATED state should not contribute towards list of
- // containers while calculating container threshold in SCMChillNodeManager
- testContainerThreshold(containers.subList(0, 10), 0.4);
- assertTrue(scmChillModeManager.getInChillMode());
-
- // When remaining 15 OPEN containers are reported by DNs, the container
- // threshold should be (10+15)/25.
- testContainerThreshold(containers.subList(10, 25), 1.0);
-
- GenericTestUtils.waitFor(() -> {
- return !scmChillModeManager.getInChillMode();
- }, 100, 1000 * 5);
- }
-
- private void testChillModeDataNodes(int numOfDns) throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration(config);
- conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
- scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
- queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
- // Assert SCM is in Chill mode.
- assertTrue(scmChillModeManager.getInChillMode());
-
- // Register all DataNodes except last one and assert SCM is in chill mode.
- for (int i = 0; i < numOfDns-1; i++) {
- queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- HddsTestUtils.createNodeRegistrationContainerReport(containers));
- assertTrue(scmChillModeManager.getInChillMode());
- assertTrue(scmChillModeManager.getCurrentContainerThreshold() == 1);
- }
-
- if(numOfDns == 0){
- GenericTestUtils.waitFor(() -> {
- return scmChillModeManager.getInChillMode();
- }, 10, 1000 * 10);
- return;
- }
- // Register last DataNode and check that SCM is out of Chill mode.
- queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- HddsTestUtils.createNodeRegistrationContainerReport(containers));
- GenericTestUtils.waitFor(() -> {
- return !scmChillModeManager.getInChillMode();
- }, 10, 1000 * 10);
- }
-
- private void testContainerThreshold(List<ContainerInfo> dnContainers,
- double expectedThreshold)
- throws Exception {
- queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- HddsTestUtils.createNodeRegistrationContainerReport(dnContainers));
- GenericTestUtils.waitFor(() -> {
- double threshold = scmChillModeManager.getCurrentContainerThreshold();
- return threshold == expectedThreshold;
- }, 100, 2000 * 9);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/addec292/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
index dfcda5f..597bcf6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
@@ -30,11 +30,11 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
+import org.apache.hadoop.hdds.scm.chillmode.SCMChillModeManager;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdds.scm.server.SCMChillModeManager;
import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.HddsDatanodeService;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/29] hadoop git commit: HDDS-810. Move the "\r\n" at the chunk end
in TestSignedChunksInputStream#singlechunkwithoutend. Contributed by Chencan.
Posted by ae...@apache.org.
HDDS-810. Move the "\r\n" at the chunk end in TestSignedChunksInputStream#singlechunkwithoutend.
Contributed by Chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00a67f77
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00a67f77
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00a67f77
Branch: refs/heads/HDDS-4
Commit: 00a67f770e86fd30dacf2cfe97824b5bcc7a7376
Parents: f0ce072
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Nov 6 07:59:40 2018 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Nov 6 10:42:19 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a67f77/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
index 27344e2..f1af7cb 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
@@ -62,7 +62,7 @@ public class TestSignedChunksInputStream {
InputStream is = fileContent("0A;chunk-signature"
+
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
- + "\n1234567890\r\n");
+ + "\n1234567890");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890", result);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/29] hadoop git commit: HDDS-798. Storage-class is showing
incorrectly. Contributed by Bharat Viswanadham.
Posted by ae...@apache.org.
HDDS-798. Storage-class is showing incorrectly. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d5cc113
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d5cc113
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d5cc113
Branch: refs/heads/HDDS-4
Commit: 3d5cc1138a07d3c0964d18250966a188ab1cf0b1
Parents: 349168c
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 6 18:19:40 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 6 18:28:54 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/ozone/client/OzoneKey.java | 16 ++++++++++++++-
.../hadoop/ozone/client/OzoneKeyDetails.java | 9 ++++++---
.../hadoop/ozone/client/rest/RestClient.java | 6 ++++--
.../hadoop/ozone/client/rpc/RpcClient.java | 14 ++++++-------
.../ozone/client/rest/response/KeyInfo.java | 20 +++++++++++++++++++
.../hadoop/ozone/web/response/KeyInfo.java | 21 ++++++++++++++++++++
.../main/smoketest/s3/objectmultidelete.robot | 4 ++++
.../ozone/s3/endpoint/BucketEndpoint.java | 9 ++++++++-
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
9 files changed, 85 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
index 7c93146..bc8135f 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.client;
+import org.apache.hadoop.hdds.client.ReplicationType;
+
/**
* A class that encapsulates OzoneKey.
*/
@@ -48,19 +50,22 @@ public class OzoneKey {
*/
private long modificationTime;
+ private ReplicationType replicationType;
+
/**
* Constructs OzoneKey from OmKeyInfo.
*
*/
public OzoneKey(String volumeName, String bucketName,
String keyName, long size, long creationTime,
- long modificationTime) {
+ long modificationTime, ReplicationType type) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.name = keyName;
this.dataSize = size;
this.creationTime = creationTime;
this.modificationTime = modificationTime;
+ this.replicationType = type;
}
/**
@@ -116,4 +121,13 @@ public class OzoneKey {
public long getModificationTime() {
return modificationTime;
}
+
+ /**
+ * Returns the replication type of the key.
+ *
+ * @return replicationType
+ */
+ public ReplicationType getReplicationType() {
+ return replicationType;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
index e7709dd..7c22f1d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.client;
+import org.apache.hadoop.hdds.client.ReplicationType;
+
import java.util.List;
/**
@@ -34,10 +36,11 @@ public class OzoneKeyDetails extends OzoneKey {
* Constructs OzoneKeyDetails from OmKeyInfo.
*/
public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
- long size, long creationTime, long modificationTime,
- List<OzoneKeyLocation> ozoneKeyLocations) {
+ long size, long creationTime, long modificationTime,
+ List<OzoneKeyLocation> ozoneKeyLocations,
+ ReplicationType type) {
super(volumeName, bucketName, keyName, size, creationTime,
- modificationTime);
+ modificationTime, type);
this.ozoneKeyLocations = ozoneKeyLocations;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 4293c08..8e106a5 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -777,7 +777,8 @@ public class RestClient implements ClientProtocol {
LOG.warn("Parse exception in getting creation time for volume", e);
}
return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(),
- keyInfo.getSize(), creationTime, modificationTime);
+ keyInfo.getSize(), creationTime, modificationTime,
+ ReplicationType.valueOf(keyInfo.getType().toString()));
}).collect(Collectors.toList());
} catch (URISyntaxException e) {
throw new IOException(e);
@@ -812,7 +813,8 @@ public class RestClient implements ClientProtocol {
keyInfo.getSize(),
HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()),
HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()),
- ozoneKeyLocations);
+ ozoneKeyLocations, ReplicationType.valueOf(
+ keyInfo.getType().toString()));
EntityUtils.consume(response);
return key;
} catch (URISyntaxException | ParseException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 5efc086..cbb2e49 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -537,7 +537,8 @@ public class RpcClient implements ClientProtocol {
key.getKeyName(),
key.getDataSize(),
key.getCreationTime(),
- key.getModificationTime()))
+ key.getModificationTime(),
+ ReplicationType.valueOf(key.getType().toString())))
.collect(Collectors.toList());
}
@@ -559,13 +560,10 @@ public class RpcClient implements ClientProtocol {
keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().forEach(
(a) -> ozoneKeyLocations.add(new OzoneKeyLocation(a.getContainerID(),
a.getLocalID(), a.getLength(), a.getOffset())));
- return new OzoneKeyDetails(keyInfo.getVolumeName(),
- keyInfo.getBucketName(),
- keyInfo.getKeyName(),
- keyInfo.getDataSize(),
- keyInfo.getCreationTime(),
- keyInfo.getModificationTime(),
- ozoneKeyLocations);
+ return new OzoneKeyDetails(keyInfo.getVolumeName(), keyInfo.getBucketName(),
+ keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
+ keyInfo.getModificationTime(), ozoneKeyLocations, ReplicationType
+ .valueOf(keyInfo.getType().toString()));
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
index 61c2abb..299826f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdds.client.ReplicationType;
/**
* KeyInfo class is used used for parsing json response
@@ -40,6 +41,25 @@ public class KeyInfo implements Comparable<KeyInfo> {
private String modifiedOn;
private long size;
private String keyName;
+ private ReplicationType type;
+
+ /**
+ * Return replication type of the key.
+ *
+ * @return replication type
+ */
+ public ReplicationType getType() {
+ return type;
+ }
+
+ /**
+ * Set replication type of the key.
+ *
+ * @param replicationType
+ */
+ public void setType(ReplicationType replicationType) {
+ this.type = replicationType;
+ }
/**
* When this key was created.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
index ba47bee..f480194 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -75,6 +76,26 @@ public class KeyInfo implements Comparable<KeyInfo> {
private String dataFileName;
+ private ReplicationType type;
+
+ /**
+ * Return replication type of the key.
+ *
+ * @return replication type
+ */
+ public ReplicationType getType() {
+ return type;
+ }
+
+ /**
+ * Set replication type of the key.
+ *
+ * @param replicationType
+ */
+ public void setType(ReplicationType replicationType) {
+ this.type = replicationType;
+ }
+
/**
* When this key was created.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
index e8b7bd3..dae3a8a 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot
@@ -36,9 +36,13 @@ Delete file with multi delete
Should contain ${result} multidelete/f1
Should contain ${result} multidelete/f2
Should contain ${result} multidelete/f3
+ Should contain ${result} REDUCED_REDUNDANCY
+ Should not contain ${result} STANDARD
${result} = Execute AWSS3APICli delete-objects --bucket ${BUCKET} --delete 'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]'
Should not contain ${result} Error
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/
Should not contain ${result} multidelete/f1
Should not contain ${result} multidelete/f2
Should contain ${result} multidelete/f3
+ Should contain ${result} REDUCED_REDUNDANCY
+ Should not contain ${result} STANDARD
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index bfbbb33..1f0764b 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -37,6 +37,7 @@ import java.io.InputStream;
import java.time.Instant;
import java.util.Iterator;
+import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata;
@@ -48,6 +49,7 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.s3.util.S3StorageType;
import org.apache.hadoop.ozone.s3.util.S3utils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
@@ -296,7 +298,12 @@ public class BucketEndpoint extends EndpointBase {
keyMetadata.setKey(next.getName());
keyMetadata.setSize(next.getDataSize());
keyMetadata.setETag("" + next.getModificationTime());
- keyMetadata.setStorageClass("STANDARD");
+ if (next.getReplicationType().toString().equals(ReplicationType
+ .STAND_ALONE.toString())) {
+ keyMetadata.setStorageClass(S3StorageType.REDUCED_REDUNDANCY.toString());
+ } else {
+ keyMetadata.setStorageClass(S3StorageType.STANDARD.toString());
+ }
keyMetadata.setLastModified(Instant.ofEpochMilli(
next.getModificationTime()));
response.addKey(keyMetadata);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5cc113/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 918e9d0..040d6df 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -93,7 +93,7 @@ public class OzoneBucketStub extends OzoneBucket {
size,
System.currentTimeMillis(),
System.currentTimeMillis(),
- new ArrayList<>()
+ new ArrayList<>(), type
));
super.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/29] hadoop git commit: HDDS-802. Container State Manager should
get open pipelines for allocating container. Contributed by Lokesh Jain.
Posted by ae...@apache.org.
HDDS-802. Container State Manager should get open pipelines for allocating container. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9317a61f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9317a61f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9317a61f
Branch: refs/heads/HDDS-4
Commit: 9317a61f3cdc5ca91c6934eec9898cee3d65441a
Parents: c80f753
Author: Yiqun Lin <yq...@apache.org>
Authored: Thu Nov 8 23:41:43 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Thu Nov 8 23:41:43 2018 +0800
----------------------------------------------------------------------
.../scm/container/ContainerStateManager.java | 4 +-
.../hdds/scm/pipeline/PipelineManager.java | 3 +
.../hdds/scm/pipeline/PipelineStateManager.java | 5 ++
.../hdds/scm/pipeline/PipelineStateMap.java | 22 +++++++
.../hdds/scm/pipeline/SCMPipelineManager.java | 11 ++++
.../scm/pipeline/TestPipelineStateManager.java | 61 ++++++++++++++++++--
6 files changed, 100 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9317a61f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
index 87505c3..74c8dcb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
@@ -248,8 +248,8 @@ public class ContainerStateManager {
try {
pipeline = pipelineManager.createPipeline(type, replicationFactor);
} catch (IOException e) {
- final List<Pipeline> pipelines =
- pipelineManager.getPipelines(type, replicationFactor);
+ final List<Pipeline> pipelines = pipelineManager
+ .getPipelines(type, replicationFactor, Pipeline.PipelineState.OPEN);
if (pipelines.isEmpty()) {
throw new IOException("Could not allocate container");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9317a61f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
index 04ec535..cce09f3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java
@@ -46,6 +46,9 @@ public interface PipelineManager extends Closeable {
List<Pipeline> getPipelines(ReplicationType type,
ReplicationFactor factor);
+ List<Pipeline> getPipelines(ReplicationType type,
+ ReplicationFactor factor, Pipeline.PipelineState state);
+
void addContainerToPipeline(PipelineID pipelineID, ContainerID containerID)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9317a61f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
index 67f74d3..9f95378 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java
@@ -64,6 +64,11 @@ class PipelineStateManager {
return pipelineStateMap.getPipelines(type, factor);
}
+ List<Pipeline> getPipelines(ReplicationType type, ReplicationFactor factor,
+ PipelineState state) {
+ return pipelineStateMap.getPipelines(type, factor, state);
+ }
+
List<Pipeline> getPipelines(ReplicationType type, PipelineState... states) {
return pipelineStateMap.getPipelines(type, states);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9317a61f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
index 7b69491..85790b2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java
@@ -166,6 +166,28 @@ class PipelineStateMap {
}
/**
+ * Get list of pipeline corresponding to specified replication type,
+ * replication factor and pipeline state.
+ *
+ * @param type - ReplicationType
+ * @param state - Required PipelineState
+ * @return List of pipelines with specified replication type,
+ * replication factor and pipeline state
+ */
+ List<Pipeline> getPipelines(ReplicationType type, ReplicationFactor factor,
+ PipelineState state) {
+ Preconditions.checkNotNull(type, "Replication type cannot be null");
+ Preconditions.checkNotNull(factor, "Replication factor cannot be null");
+ Preconditions.checkNotNull(state, "Pipeline state cannot be null");
+
+ return pipelineMap.values().stream().filter(
+ pipeline -> pipeline.getType() == type
+ && pipeline.getPipelineState() == state
+ && pipeline.getFactor() == factor)
+ .collect(Collectors.toList());
+ }
+
+ /**
* Get set of containerIDs corresponding to a pipeline.
*
* @param pipelineID - PipelineID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9317a61f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
index 1c21748..5e8d0dc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java
@@ -166,6 +166,17 @@ public class SCMPipelineManager implements PipelineManager {
}
@Override
+ public List<Pipeline> getPipelines(ReplicationType type,
+ ReplicationFactor factor, Pipeline.PipelineState state) {
+ lock.readLock().lock();
+ try {
+ return stateManager.getPipelines(type, factor, state);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
public void addContainerToPipeline(PipelineID pipelineID,
ContainerID containerID) throws IOException {
lock.writeLock().lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9317a61f/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
index fd6f76b..0f5692e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java
@@ -157,8 +157,8 @@ public class TestPipelineStateManager {
stateManager.getPipelines(type, factor);
Assert.assertEquals(15, pipelines1.size());
pipelines1.stream().forEach(p -> {
- Assert.assertEquals(p.getType(), type);
- Assert.assertEquals(p.getFactor(), factor);
+ Assert.assertEquals(type, p.getType());
+ Assert.assertEquals(factor, p.getFactor());
});
}
}
@@ -203,8 +203,8 @@ public class TestPipelineStateManager {
.getPipelines(type, Pipeline.PipelineState.OPEN);
Assert.assertEquals(5, pipelines1.size());
pipelines1.forEach(p -> {
- Assert.assertEquals(p.getType(), type);
- Assert.assertEquals(p.getPipelineState(), Pipeline.PipelineState.OPEN);
+ Assert.assertEquals(type, p.getType());
+ Assert.assertEquals(Pipeline.PipelineState.OPEN, p.getPipelineState());
});
pipelines1 = stateManager
@@ -220,6 +220,59 @@ public class TestPipelineStateManager {
}
@Test
+ public void testGetPipelinesByTypeFactorAndState() throws IOException {
+ Set<Pipeline> pipelines = new HashSet<>();
+ for (HddsProtos.ReplicationType type : HddsProtos.ReplicationType
+ .values()) {
+ for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor
+ .values()) {
+ for (int i = 0; i < 5; i++) {
+ // 5 pipelines in allocated state for each type and factor
+ Pipeline pipeline =
+ createDummyPipeline(type, factor, factor.getNumber());
+ stateManager.addPipeline(pipeline);
+ pipelines.add(pipeline);
+
+ // 5 pipelines in open state for each type and factor
+ pipeline = createDummyPipeline(type, factor, factor.getNumber());
+ stateManager.addPipeline(pipeline);
+ stateManager.openPipeline(pipeline.getId());
+ pipelines.add(pipeline);
+
+ // 5 pipelines in closed state for each type and factor
+ pipeline = createDummyPipeline(type, factor, factor.getNumber());
+ stateManager.addPipeline(pipeline);
+ stateManager.finalizePipeline(pipeline.getId());
+ pipelines.add(pipeline);
+ }
+ }
+ }
+
+ for (HddsProtos.ReplicationType type : HddsProtos.ReplicationType
+ .values()) {
+ for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor
+ .values()) {
+ for (Pipeline.PipelineState state : Pipeline.PipelineState.values()) {
+ // verify pipelines received
+ List<Pipeline> pipelines1 =
+ stateManager.getPipelines(type, factor, state);
+ Assert.assertEquals(5, pipelines1.size());
+ pipelines1.forEach(p -> {
+ Assert.assertEquals(type, p.getType());
+ Assert.assertEquals(factor, p.getFactor());
+ Assert.assertEquals(state, p.getPipelineState());
+ });
+ }
+ }
+ }
+
+ //clean up
+ for (Pipeline pipeline : pipelines) {
+ removePipeline(pipeline);
+ }
+ }
+
+ @Test
public void testAddAndGetContainer() throws IOException {
long containerID = 0;
Pipeline pipeline = createDummyPipeline(1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/29] hadoop git commit: YARN-8970. Improve the debug message in
CS#allocateContainerOnSingleNode. Contributed by Zhankun Tang.
Posted by ae...@apache.org.
YARN-8970. Improve the debug message in CS#allocateContainerOnSingleNode. Contributed by Zhankun Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d6554c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d6554c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d6554c7
Branch: refs/heads/HDDS-4
Commit: 5d6554c722f08f79bce904e021243605ee75bae3
Parents: ffc9c50
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Nov 6 14:50:09 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Nov 6 14:50:09 2018 +0800
----------------------------------------------------------------------
.../resourcemanager/scheduler/capacity/CapacityScheduler.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d6554c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index c5ad2ce..5d7f1ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1615,8 +1615,8 @@ public class CapacityScheduler extends
.add(node.getUnallocatedResource(), node.getTotalKillableResources()),
minimumAllocation) <= 0) {
if (LOG.isDebugEnabled()) {
- LOG.debug("This node or this node partition doesn't have available or"
- + "killable resource");
+ LOG.debug("This node or node partition doesn't have available or" +
+ " preemptible resource");
}
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/29] hadoop git commit: YARN-8976. Remove redundant modifiers in
interface ApplicationConstants. Contributed by Zhankun Tang.
Posted by ae...@apache.org.
YARN-8976. Remove redundant modifiers in interface ApplicationConstants. Contributed by Zhankun Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/482716e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/482716e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/482716e5
Branch: refs/heads/HDDS-4
Commit: 482716e5a4d1edfd3aa6a1ae65a58652f89375f1
Parents: 951c98f
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Nov 7 10:48:07 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Nov 7 10:48:07 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/api/ApplicationConstants.java | 22 ++++++++++----------
1 file changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/482716e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index eb03fb2..f5d8f02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -39,12 +39,12 @@ public interface ApplicationConstants {
* The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
* only
*/
- public static final String APP_SUBMIT_TIME_ENV = "APP_SUBMIT_TIME_ENV";
+ String APP_SUBMIT_TIME_ENV = "APP_SUBMIT_TIME_ENV";
/**
* The cache file into which container token is written
*/
- public static final String CONTAINER_TOKEN_FILE_ENV_NAME =
+ String CONTAINER_TOKEN_FILE_ENV_NAME =
UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
/**
@@ -74,14 +74,14 @@ public interface ApplicationConstants {
* ApplicationMaster's environment only. This states that for all non-relative
* web URLs in the app masters web UI what base should they have.
*/
- public static final String APPLICATION_WEB_PROXY_BASE_ENV =
+ String APPLICATION_WEB_PROXY_BASE_ENV =
"APPLICATION_WEB_PROXY_BASE";
/**
* The temporary environmental variable for container log directory. This
* should be replaced by real container log directory on container launch.
*/
- public static final String LOG_DIR_EXPANSION_VAR = "<LOG_DIR>";
+ String LOG_DIR_EXPANSION_VAR = "<LOG_DIR>";
/**
* This constant is used to construct class path and it will be replaced with
@@ -92,7 +92,7 @@ public interface ApplicationConstants {
*/
@Public
@Unstable
- public static final String CLASS_PATH_SEPARATOR= "<CPS>";
+ String CLASS_PATH_SEPARATOR= "<CPS>";
/**
* The following two constants are used to expand parameter and it will be
@@ -105,7 +105,7 @@ public interface ApplicationConstants {
*/
@Public
@Unstable
- public static final String PARAMETER_EXPANSION_LEFT="{{";
+ String PARAMETER_EXPANSION_LEFT="{{";
/**
* User has to use this constant to construct class path if user wants
@@ -114,11 +114,11 @@ public interface ApplicationConstants {
*/
@Public
@Unstable
- public static final String PARAMETER_EXPANSION_RIGHT="}}";
+ String PARAMETER_EXPANSION_RIGHT="}}";
- public static final String STDERR = "stderr";
+ String STDERR = "stderr";
- public static final String STDOUT = "stdout";
+ String STDOUT = "stdout";
/**
* The type of launch for the container.
@@ -136,7 +136,7 @@ public interface ApplicationConstants {
* Some of the environment variables for applications are <em>final</em>
* i.e. they cannot be modified by the applications.
*/
- public enum Environment {
+ enum Environment {
/**
* $USER
* Final, non-modifiable.
@@ -283,7 +283,7 @@ public interface ApplicationConstants {
"YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE");
private final String variable;
- private Environment(String variable) {
+ Environment(String variable) {
this.variable = variable;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/29] hadoop git commit: YARN-8957. Add Serializable interface to
ComponentContainers. Contributed by Zhankun Tang
Posted by ae...@apache.org.
YARN-8957. Add Serializable interface to ComponentContainers.
Contributed by Zhankun Tang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08d69d91
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08d69d91
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08d69d91
Branch: refs/heads/HDDS-4
Commit: 08d69d91f2f09a3ee7711f9d56a535787b30ffd2
Parents: 887244d
Author: Eric Yang <ey...@apache.org>
Authored: Tue Nov 6 14:46:05 2018 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Nov 6 14:46:05 2018 -0500
----------------------------------------------------------------------
.../hadoop/yarn/service/api/records/ComponentContainers.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08d69d91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentContainers.java
index 9ef8b9f..7c8ab4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentContainers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentContainers.java
@@ -24,6 +24,7 @@ import io.swagger.annotations.ApiModelProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.io.Serializable;
import java.util.List;
/**
@@ -33,7 +34,7 @@ import java.util.List;
@InterfaceStability.Unstable
@ApiModel(description = "Containers of a component.")
@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ComponentContainers {
+public class ComponentContainers implements Serializable {
private static final long serialVersionUID = -1456748479118874991L;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/29] hadoop git commit: HADOOP-15907. Add missing maven modules in
BUILDING.txt. Contributed Wanqiang Ji.
Posted by ae...@apache.org.
HADOOP-15907. Add missing maven modules in BUILDING.txt. Contributed Wanqiang Ji.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6444f1c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6444f1c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6444f1c
Branch: refs/heads/HDDS-4
Commit: e6444f1c640cda86e1c97fbfebf68de92a162c95
Parents: addec29
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Nov 7 16:45:16 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Nov 7 16:45:16 2018 +0800
----------------------------------------------------------------------
BUILDING.txt | 30 +++++++++++++++++++-----------
1 file changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6444f1c/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 9727004..d35e3af 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -93,17 +93,25 @@ Optional packages:
----------------------------------------------------------------------------------
Maven main modules:
- hadoop (Main Hadoop project)
- - hadoop-project (Parent POM for all Hadoop Maven modules. )
- (All plugins & dependencies versions are defined here.)
- - hadoop-project-dist (Parent POM for modules that generate distributions.)
- - hadoop-annotations (Generates the Hadoop doclet used to generated the Javadocs)
- - hadoop-assemblies (Maven assemblies used by the different modules)
- - hadoop-common-project (Hadoop Common)
- - hadoop-hdfs-project (Hadoop HDFS)
- - hadoop-mapreduce-project (Hadoop MapReduce)
- - hadoop-tools (Hadoop tools like Streaming, Distcp, etc.)
- - hadoop-dist (Hadoop distribution assembler)
+ hadoop (Main Hadoop project)
+ - hadoop-project (Parent POM for all Hadoop Maven modules. )
+ (All plugins & dependencies versions are defined here.)
+ - hadoop-project-dist (Parent POM for modules that generate distributions.)
+ - hadoop-annotations (Generates the Hadoop doclet used to generated the Javadocs)
+ - hadoop-assemblies (Maven assemblies used by the different modules)
+ - hadoop-maven-plugins (Maven plugins used in project)
+ - hadoop-build-tools (Build tools like checkstyle, etc.)
+ - hadoop-common-project (Hadoop Common)
+ - hadoop-hdfs-project (Hadoop HDFS)
+ - hadoop-yarn-project (Hadoop YARN)
+ - hadoop-mapreduce-project (Hadoop MapReduce)
+ - hadoop-ozone (Hadoop Ozone)
+ - hadoop-hdds (Hadoop Distributed Data Store)
+ - hadoop-tools (Hadoop tools like Streaming, Distcp, etc.)
+ - hadoop-dist (Hadoop distribution assembler)
+ - hadoop-client-modules (Hadoop client modules)
+ - hadoop-minicluster (Hadoop minicluster artifacts)
+ - hadoop-cloud-storage-project (Generates artifacts to access cloud storage like aws, azure, etc.)
----------------------------------------------------------------------------------
Where to run Maven from?
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/29] hadoop git commit: YARN-8977. Remove unnecessary type casting
when calling AbstractYarnScheduler#getSchedulerNode. Contributed by Wanqiang
Ji.
Posted by ae...@apache.org.
YARN-8977. Remove unnecessary type casting when calling AbstractYarnScheduler#getSchedulerNode. Contributed by Wanqiang Ji.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c96cbe86
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c96cbe86
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c96cbe86
Branch: refs/heads/HDDS-4
Commit: c96cbe8659587cfc114a96aab1be5cc85029fe44
Parents: 0b6625a
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Nov 7 22:27:57 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Nov 7 22:42:22 2018 +0800
----------------------------------------------------------------------
.../resourcemanager/scheduler/capacity/CapacityScheduler.java | 6 +++---
.../scheduler/fair/TestContinuousScheduling.java | 3 +--
.../resourcemanager/scheduler/fair/TestFairScheduler.java | 3 +--
3 files changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c96cbe86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e604b81..618ee20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2192,8 +2192,8 @@ public class CapacityScheduler extends
super.completedContainer(killableContainer, SchedulerUtils
.createPreemptedContainerStatus(killableContainer.getContainerId(),
SchedulerUtils.PREEMPTED_CONTAINER), RMContainerEventType.KILL);
- } else{
- FiCaSchedulerNode node = (FiCaSchedulerNode) getSchedulerNode(
+ } else {
+ FiCaSchedulerNode node = getSchedulerNode(
killableContainer.getAllocatedNode());
FiCaSchedulerApp application = getCurrentAttemptForContainer(
@@ -2225,7 +2225,7 @@ public class CapacityScheduler extends
+ nonKillableContainer.toString());
}
- FiCaSchedulerNode node = (FiCaSchedulerNode) getSchedulerNode(
+ FiCaSchedulerNode node = getSchedulerNode(
nonKillableContainer.getAllocatedNode());
FiCaSchedulerApp application = getCurrentAttemptForContainer(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c96cbe86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 2512787..e6a841a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -376,8 +376,7 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
- FSSchedulerNode node =
- (FSSchedulerNode) scheduler.getSchedulerNode(node1.getNodeID());
+ FSSchedulerNode node = scheduler.getSchedulerNode(node1.getNodeID());
// Tick the time and let the fsApp startTime different from initScheduler
// time
mockClock.tickSec(delayThresholdTimeMs / 1000);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c96cbe86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index dbf4b2b..0d6caeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4557,8 +4557,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
- FSSchedulerNode node = (FSSchedulerNode)scheduler.getSchedulerNode(
- node1.getNodeID());
+ FSSchedulerNode node = scheduler.getSchedulerNode(node1.getNodeID());
NodeRemovedSchedulerEvent removeNode1 =
new NodeRemovedSchedulerEvent(node1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/29] hadoop git commit: HDDS-737. Introduce Incremental Container
Report. Contributed by Nanda kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index c1409cb..5f419d3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -39,10 +39,10 @@ import org.apache.hadoop.hdds.scm.block.PendingDeleteHandler;
import org.apache.hadoop.hdds.scm.chillmode.SCMChillModeManager;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
-import org.apache.hadoop.hdds.scm.container.CloseContainerWatcher;
import org.apache.hadoop.hdds.scm.container.ContainerActionsHandler;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
+import org.apache.hadoop.hdds.scm.container.IncrementalContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.replication
@@ -221,7 +221,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
CommandStatusReportHandler cmdStatusReportHandler =
new CommandStatusReportHandler();
- NewNodeHandler newNodeHandler = new NewNodeHandler(scmNodeManager);
+ NewNodeHandler newNodeHandler = new NewNodeHandler();
StaleNodeHandler staleNodeHandler =
new StaleNodeHandler(scmNodeManager, pipelineManager);
DeadNodeHandler deadNodeHandler = new DeadNodeHandler(scmNodeManager,
@@ -231,8 +231,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService());
ContainerReportHandler containerReportHandler =
- new ContainerReportHandler(containerManager, scmNodeManager,
- replicationStatus);
+ new ContainerReportHandler(scmNodeManager, pipelineManager,
+ containerManager, replicationStatus);
+
+ IncrementalContainerReportHandler incrementalContainerReportHandler =
+ new IncrementalContainerReportHandler(
+ pipelineManager, containerManager);
PipelineActionHandler pipelineActionHandler =
new PipelineActionHandler(pipelineManager);
@@ -258,13 +262,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
replicationManager = new ReplicationManager(containerPlacementPolicy,
containerManager, eventQueue, commandWatcherLeaseManager);
- // setup CloseContainer watcher
- CloseContainerWatcher closeContainerWatcher =
- new CloseContainerWatcher(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- SCMEvents.CLOSE_CONTAINER_STATUS, commandWatcherLeaseManager,
- containerManager);
- closeContainerWatcher.start(eventQueue);
-
scmAdminUsernames = conf.getTrimmedStringCollection(OzoneConfigKeys
.OZONE_ADMINISTRATORS);
scmUsername = UserGroupInformation.getCurrentUser().getUserName();
@@ -282,6 +279,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
eventQueue.addHandler(SCMEvents.CONTAINER_REPORT, containerReportHandler);
+ eventQueue.addHandler(SCMEvents.INCREMENTAL_CONTAINER_REPORT,
+ incrementalContainerReportHandler);
eventQueue.addHandler(SCMEvents.CONTAINER_ACTIONS, actionsHandler);
eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
eventQueue.addHandler(SCMEvents.NEW_NODE, newNodeHandler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
index afa25e2..279acf0 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.UUID;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
@@ -51,8 +50,6 @@ public class TestCommandStatusReportHandler implements EventPublisher {
private static final Logger LOG = LoggerFactory
.getLogger(TestCommandStatusReportHandler.class);
private CommandStatusReportHandler cmdStatusReportHandler;
- private String storagePath = GenericTestUtils.getRandomizedTempPath()
- .concat("/" + UUID.randomUUID().toString());
@Before
public void setup() {
@@ -68,8 +65,6 @@ public class TestCommandStatusReportHandler implements EventPublisher {
.emptyList());
cmdStatusReportHandler.onMessage(report, this);
assertFalse(logCapturer.getOutput().contains("Delete_Block_Status"));
- assertFalse(logCapturer.getOutput().contains(
- "Close_Container_Command_Status"));
assertFalse(logCapturer.getOutput().contains("Replicate_Command_Status"));
@@ -78,13 +73,9 @@ public class TestCommandStatusReportHandler implements EventPublisher {
assertTrue(logCapturer.getOutput().contains("firing event of type " +
"Delete_Block_Status"));
assertTrue(logCapturer.getOutput().contains("firing event of type " +
- "Close_Container_Command_Status"));
- assertTrue(logCapturer.getOutput().contains("firing event of type " +
"Replicate_Command_Status"));
assertTrue(logCapturer.getOutput().contains("type: " +
- "closeContainerCommand"));
- assertTrue(logCapturer.getOutput().contains("type: " +
"deleteBlocksCommand"));
assertTrue(logCapturer.getOutput().contains("type: " +
"replicateContainerCommand"));
@@ -119,11 +110,6 @@ public class TestCommandStatusReportHandler implements EventPublisher {
.setType(Type.deleteBlocksCommand);
reports.add(builder.build());
- builder.setCmdId(HddsIdFactory.getLongId())
- .setStatus(CommandStatus.Status.EXECUTED)
- .setType(Type.closeContainerCommand);
- reports.add(builder.build());
-
builder.setMsg("Not enough space")
.setCmdId(HddsIdFactory.getLongId())
.setStatus(CommandStatus.Status.FAILED)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 8d36d29..d5e68a8 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.VersionResponse;
@@ -44,6 +43,7 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.assertj.core.util.Preconditions;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -139,18 +139,6 @@ public class MockNodeManager implements NodeManager {
}
/**
- * Removes a data node from the management of this Node Manager.
- *
- * @param node - DataNode.
- * @throws NodeNotFoundException
- */
- @Override
- public void removeNode(DatanodeDetails node)
- throws NodeNotFoundException {
-
- }
-
- /**
* Gets all Live Datanodes that is currently communicating with SCM.
*
* @param nodestate - State of the node
@@ -248,8 +236,8 @@ public class MockNodeManager implements NodeManager {
* @return Set of PipelineID
*/
@Override
- public Set<PipelineID> getPipelineByDnID(UUID dnId) {
- return node2PipelineMap.getPipelines(dnId);
+ public Set<PipelineID> getPipelines(DatanodeDetails dnId) {
+ return node2PipelineMap.getPipelines(dnId.getUuid());
}
/**
@@ -290,7 +278,8 @@ public class MockNodeManager implements NodeManager {
* @param nodeReport
*/
@Override
- public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) {
+ public void processNodeReport(DatanodeDetails dnUuid,
+ NodeReportProto nodeReport) {
// do nothing
}
@@ -302,21 +291,13 @@ public class MockNodeManager implements NodeManager {
* addDatanodeInContainerMap call.
*/
@Override
- public void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
- throws SCMException {
- node2ContainerMap.setContainersForDatanode(uuid, containerIds);
- }
-
- /**
- * Process containerReport received from datanode.
- * @param uuid - DataonodeID
- * @param containerIds - Set of containerIDs
- * @return The result after processing containerReport
- */
- @Override
- public ReportResult<ContainerID> processContainerReport(UUID uuid,
- Set<ContainerID> containerIds) {
- return node2ContainerMap.processReport(uuid, containerIds);
+ public void setContainers(DatanodeDetails uuid, Set<ContainerID> containerIds)
+ throws NodeNotFoundException {
+ try {
+ node2ContainerMap.setContainersForDatanode(uuid.getUuid(), containerIds);
+ } catch (SCMException e) {
+ throw new NodeNotFoundException(e.getMessage());
+ }
}
/**
@@ -325,21 +306,8 @@ public class MockNodeManager implements NodeManager {
* @return - set of containerIDs
*/
@Override
- public Set<ContainerID> getContainers(UUID uuid) {
- return node2ContainerMap.getContainers(uuid);
- }
-
- /**
- * Insert a new datanode with set of containerIDs for containers available
- * on it.
- * @param uuid - DatanodeID
- * @param containerIDs - Set of ContainerIDs
- * @throws SCMException - if datanode already exists
- */
- @Override
- public void addDatanodeInContainerMap(UUID uuid,
- Set<ContainerID> containerIDs) throws SCMException {
- node2ContainerMap.insertNewDatanode(uuid, containerIDs);
+ public Set<ContainerID> getContainers(DatanodeDetails uuid) {
+ return node2ContainerMap.getContainers(uuid.getUuid());
}
// Returns the number of commands that is queued to this node manager.
@@ -393,6 +361,12 @@ public class MockNodeManager implements NodeManager {
@Override
public RegisteredCommand register(DatanodeDetails datanodeDetails,
NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto) {
+ try {
+ node2ContainerMap.insertNewDatanode(datanodeDetails.getUuid(),
+ Collections.emptySet());
+ } catch (SCMException e) {
+ e.printStackTrace();
+ }
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index faae6f5..a2eba92 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.replication
.ReplicationActivityStatus;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
@@ -69,7 +70,7 @@ public class TestContainerReportHandler implements EventPublisher {
//TODO: Rewrite it
@Ignore
@Test
- public void test() throws IOException {
+ public void test() throws IOException, NodeNotFoundException {
String testDir = GenericTestUtils.getTempPath(
this.getClass().getSimpleName());
//GIVEN
@@ -85,17 +86,17 @@ public class TestContainerReportHandler implements EventPublisher {
new ReplicationActivityStatus();
ContainerReportHandler reportHandler =
- new ContainerReportHandler(containerManager, nodeManager,
- replicationActivityStatus);
+ new ContainerReportHandler(nodeManager, pipelineManager,
+ containerManager, replicationActivityStatus);
DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
DatanodeDetails dn2 = TestUtils.randomDatanodeDetails();
DatanodeDetails dn3 = TestUtils.randomDatanodeDetails();
DatanodeDetails dn4 = TestUtils.randomDatanodeDetails();
- nodeManager.addDatanodeInContainerMap(dn1.getUuid(), new HashSet<>());
- nodeManager.addDatanodeInContainerMap(dn2.getUuid(), new HashSet<>());
- nodeManager.addDatanodeInContainerMap(dn3.getUuid(), new HashSet<>());
- nodeManager.addDatanodeInContainerMap(dn4.getUuid(), new HashSet<>());
+ nodeManager.setContainers(dn1, new HashSet<>());
+ nodeManager.setContainers(dn2, new HashSet<>());
+ nodeManager.setContainers(dn3, new HashSet<>());
+ nodeManager.setContainers(dn4, new HashSet<>());
ContainerInfo cont1 = containerManager
.allocateContainer(ReplicationType.STAND_ALONE,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
index 446eb58..5c4617c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
@@ -24,6 +24,8 @@ import java.util.Set;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -96,6 +98,7 @@ public class TestContainerStateManager {
throws ContainerNotFoundException {
ContainerReplica replica = ContainerReplica.newBuilder()
.setContainerID(cont.containerID())
+ .setContainerState(ContainerReplicaProto.State.CLOSED)
.setDatanodeDetails(node)
.build();
containerStateManager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
index be4cce6..fad67b8 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -30,8 +29,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -47,8 +44,6 @@ import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
@@ -187,10 +182,12 @@ public class TestSCMContainerManager {
containerManager.updateContainerReplica(contInfo.containerID(),
ContainerReplica.newBuilder().setContainerID(contInfo.containerID())
- .setDatanodeDetails(dn1).build());
+ .setContainerState(ContainerReplicaProto.State.CLOSED)
+ .setDatanodeDetails(dn1).build());
containerManager.updateContainerReplica(contInfo.containerID(),
ContainerReplica.newBuilder().setContainerID(contInfo.containerID())
- .setDatanodeDetails(dn2).build());
+ .setContainerState(ContainerReplicaProto.State.CLOSED)
+ .setDatanodeDetails(dn2).build());
Assert.assertEquals(2,
containerManager.getContainerReplicas(
@@ -240,102 +237,6 @@ public class TestSCMContainerManager {
HddsProtos.LifeCycleEvent.CREATED);
}
- @Test
- public void testFullContainerReport() throws Exception {
- ContainerInfo info = createContainer();
- DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- List<ContainerReplicaProto> reports =
- new ArrayList<>();
- ContainerReplicaProto.Builder ciBuilder =
- ContainerReplicaProto.newBuilder();
- ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
- .setSize(5368709120L)
- .setUsed(2000000000L)
- .setKeyCount(100000000L)
- .setReadCount(100000000L)
- .setWriteCount(100000000L)
- .setReadBytes(2000000000L)
- .setWriteBytes(2000000000L)
- .setContainerID(info.getContainerID())
- .setState(ContainerReplicaProto.State.CLOSED)
- .setDeleteTransactionId(0);
-
- reports.add(ciBuilder.build());
-
- ContainerReportsProto.Builder crBuilder = ContainerReportsProto
- .newBuilder();
- crBuilder.addAllReports(reports);
-
- containerManager.processContainerReports(
- datanodeDetails, crBuilder.build());
-
- ContainerInfo updatedContainer =
- containerManager.getContainer(info.containerID());
- Assert.assertEquals(100000000L,
- updatedContainer.getNumberOfKeys());
- Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes());
-
- for (ContainerReplicaProto c : reports) {
- Assert.assertEquals(containerManager.getContainerReplicas(
- ContainerID.valueof(c.getContainerID())).size(), 1);
- }
-
- containerManager.processContainerReports(TestUtils.randomDatanodeDetails(),
- crBuilder.build());
- for (ContainerReplicaProto c : reports) {
- Assert.assertEquals(containerManager.getContainerReplicas(
- ContainerID.valueof(c.getContainerID())).size(), 2);
- }
- }
-
- @Test
- public void testListContainerAfterReport() throws Exception {
- ContainerInfo info1 = createContainer();
- ContainerInfo info2 = createContainer();
- DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- List<ContainerReplicaProto> reports =
- new ArrayList<>();
- ContainerReplicaProto.Builder ciBuilder =
- ContainerReplicaProto.newBuilder();
- long cID1 = info1.getContainerID();
- long cID2 = info2.getContainerID();
- ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
- .setSize(1000000000L)
- .setUsed(987654321L)
- .setKeyCount(100000000L)
- .setReadBytes(1000000000L)
- .setWriteBytes(1000000000L)
- .setContainerID(cID1)
- .setState(ContainerReplicaProto.State.CLOSED);
- reports.add(ciBuilder.build());
-
- ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9")
- .setSize(1000000000L)
- .setUsed(123456789L)
- .setKeyCount(200000000L)
- .setReadBytes(3000000000L)
- .setWriteBytes(4000000000L)
- .setContainerID(cID2);
- reports.add(ciBuilder.build());
-
- ContainerReportsProto.Builder crBuilder = ContainerReportsProto
- .newBuilder();
- crBuilder.addAllReports(reports);
-
- containerManager.processContainerReports(
- datanodeDetails, crBuilder.build());
-
- List<ContainerInfo> list = containerManager.listContainer(
- ContainerID.valueof(1), 50);
- Assert.assertEquals(2, list.stream().filter(
- x -> x.getContainerID() == cID1 || x.getContainerID() == cID2).count());
- Assert.assertEquals(300000000L, list.stream().filter(
- x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
- .mapToLong(x -> x.getNumberOfKeys()).sum());
- Assert.assertEquals(1111111110L, list.stream().filter(
- x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
- .mapToLong(x -> x.getUsedBytes()).sum());
- }
@Test
public void testCloseContainer() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
index 571a5fb..0be279d 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
@@ -28,7 +28,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
@@ -36,8 +39,10 @@ import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationRequestToRepeat;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms
+ .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.replication
+ .ReplicationManager.ReplicationRequestToRepeat;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.lease.LeaseManager;
@@ -81,6 +86,7 @@ public class TestReplicationManager {
listOfDatanodeDetails.add(dd);
listOfContainerReplica.add(ContainerReplica.newBuilder()
.setContainerID(ContainerID.valueof(i))
+ .setContainerState(ContainerReplicaProto.State.CLOSED)
.setDatanodeDetails(dd).build());
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 51099f4..c1f2c69 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -30,8 +30,12 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
@@ -42,10 +46,11 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .NodeReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -93,7 +98,7 @@ public class TestDeadNodeHandler {
}
@Test
- public void testOnMessage() throws IOException {
+ public void testOnMessage() throws IOException, NodeNotFoundException {
//GIVEN
DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails();
DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails();
@@ -269,11 +274,9 @@ public class TestDeadNodeHandler {
container1.containerID(), HddsProtos.LifeCycleEvent.CREATED);
TestUtils.closeContainer(containerManager, container1.containerID());
- registerReplicas(dn1, container1);
-
deadNodeHandler.onMessage(dn1, eventQueue);
Assert.assertTrue(logCapturer.getOutput().contains(
- "Exception while removing container replica "));
+ "DeadNode event for a unregistered node"));
}
private void registerReplicas(ContainerManager containerManager,
@@ -283,6 +286,7 @@ public class TestDeadNodeHandler {
containerManager.updateContainerReplica(
new ContainerID(container.getContainerID()),
ContainerReplica.newBuilder()
+ .setContainerState(ContainerReplicaProto.State.OPEN)
.setContainerID(container.containerID())
.setDatanodeDetails(datanode).build());
}
@@ -290,9 +294,9 @@ public class TestDeadNodeHandler {
private void registerReplicas(DatanodeDetails datanode,
ContainerInfo... containers)
- throws SCMException {
+ throws NodeNotFoundException {
nodeManager
- .addDatanodeInContainerMap(datanode.getUuid(),
+ .setContainers(datanode,
Arrays.stream(containers)
.map(container -> new ContainerID(container.getContainerID()))
.collect(Collectors.toSet()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java
deleted file mode 100644
index ec13534..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/TestCloseContainerWatcher.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container;
-
-import org.apache.hadoop.hdds.HddsIdFactory;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.CommandStatus;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
-import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
- .CloseContainerStatus;
-import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler
- .CloseContainerRetryableReq;
-import org.apache.hadoop.hdds.scm.container.CloseContainerWatcher;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.hdds.server.events.EventWatcher;
-import org.apache.hadoop.ozone.lease.LeaseManager;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.event.Level;
-
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.when;
-
-/**
- * Test class for {@link CloseContainerWatcher}.
- * */
-public class TestCloseContainerWatcher implements EventHandler<ContainerID> {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TestCloseContainerWatcher.class);
- private static EventWatcher<CloseContainerRetryableReq, CloseContainerStatus>
- watcher;
- private static LeaseManager<Long> leaseManager;
- private static SCMContainerManager containerManager = Mockito
- .mock(SCMContainerManager.class);
- private static EventQueue queue;
- @Rule
- public Timeout timeout = new Timeout(1000*15);
-
- @After
- public void stop() {
- leaseManager.shutdown();
- queue.close();
- }
-
- /*
- * This test will test watcher for Failure status event.
- * */
- @Test
- public void testWatcherForFailureStatusEvent() throws
- InterruptedException, IOException {
- setupWatcher(90000L);
- long id1 = HddsIdFactory.getLongId();
- long id2 = HddsIdFactory.getLongId();
- queue.addHandler(SCMEvents.CLOSE_CONTAINER, this);
- setupMock(id1, id2, true);
- GenericTestUtils.LogCapturer testLogger = GenericTestUtils.LogCapturer
- .captureLogs(LOG);
- GenericTestUtils.LogCapturer watcherLogger = GenericTestUtils.LogCapturer
- .captureLogs(CloseContainerWatcher.LOG);
- GenericTestUtils.setLogLevel(CloseContainerWatcher.LOG, Level.TRACE);
- testLogger.clearOutput();
- watcherLogger.clearOutput();
-
- CommandStatus cmdStatus1 = CommandStatus.newBuilder()
- .setCmdId(id1)
- .setStatus(CommandStatus.Status.FAILED)
- .setType(Type.closeContainerCommand).build();
- CommandStatus cmdStatus2 = CommandStatus.newBuilder()
- .setCmdId(id2)
- .setStatus(CommandStatus.Status.FAILED)
- .setType(Type.closeContainerCommand).build();
-
- // File events to watcher
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id1)));
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id2)));
- Thread.sleep(10L);
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_STATUS, new
- CloseContainerStatus(cmdStatus1));
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_STATUS, new
- CloseContainerStatus(cmdStatus2));
-
- Thread.sleep(1000*4L);
- // validation
- assertTrue(watcherLogger.getOutput().contains("CloseContainerCommand for " +
- "containerId: " + id1 + " executed"));
- assertTrue(watcherLogger.getOutput().contains("CloseContainerCommand for " +
- "containerId: " + id2 + " executed"));
- assertTrue(
- testLogger.getOutput().contains("Handling closeContainerEvent " +
- "for containerId: id=" + id1));
- assertTrue(testLogger.getOutput().contains("Handling closeContainerEvent " +
- "for containerId: id=" + id2));
-
- }
-
- @Test
- public void testWatcherForPendingStatusEvent() throws
- InterruptedException, IOException {
- setupWatcher(90000L);
- long id1 = HddsIdFactory.getLongId();
- long id2 = HddsIdFactory.getLongId();
- queue.addHandler(SCMEvents.CLOSE_CONTAINER, this);
- setupMock(id1, id2, true);
- GenericTestUtils.LogCapturer testLogger = GenericTestUtils.LogCapturer
- .captureLogs(LOG);
- GenericTestUtils.LogCapturer watcherLogger = GenericTestUtils.LogCapturer
- .captureLogs(CloseContainerWatcher.LOG);
- GenericTestUtils.setLogLevel(CloseContainerWatcher.LOG, Level.TRACE);
- testLogger.clearOutput();
- watcherLogger.clearOutput();
-
- CommandStatus cmdStatus1 = CommandStatus.newBuilder()
- .setCmdId(id1)
- .setStatus(CommandStatus.Status.PENDING)
- .setType(Type.closeContainerCommand).build();
- CommandStatus cmdStatus2 = CommandStatus.newBuilder()
- .setCmdId(id2)
- .setStatus(CommandStatus.Status.PENDING)
- .setType(Type.closeContainerCommand).build();
-
- // File events to watcher
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id1)));
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id2)));
- Thread.sleep(10L);
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_STATUS, new
- CloseContainerStatus(cmdStatus1));
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_STATUS, new
- CloseContainerStatus(cmdStatus2));
-
- Thread.sleep(1000*2L);
- // validation
- assertFalse(watcherLogger.getOutput().contains("CloseContainerCommand "
- + "for containerId: " + id1 + " executed"));
- assertFalse(watcherLogger.getOutput().contains("CloseContainerCommand "
- + "for containerId: " + id2 + " executed"));
- assertFalse(testLogger.getOutput().contains("Handling "
- + "closeContainerEvent for containerId: id=" + id1));
- assertFalse(testLogger.getOutput().contains("Handling "
- + "closeContainerEvent for containerId: id=" + id2));
-
- }
-
- @Test
- public void testWatcherForExecutedStatusEvent()
- throws IOException, InterruptedException {
- setupWatcher(90000L);
- long id1 = HddsIdFactory.getLongId();
- long id2 = HddsIdFactory.getLongId();
- queue.addHandler(SCMEvents.CLOSE_CONTAINER, this);
- setupMock(id1, id2, true);
- GenericTestUtils.LogCapturer testLogger = GenericTestUtils.LogCapturer
- .captureLogs(LOG);
- GenericTestUtils.LogCapturer watcherLogger = GenericTestUtils.LogCapturer
- .captureLogs(CloseContainerWatcher.LOG);
- GenericTestUtils.setLogLevel(CloseContainerWatcher.LOG, Level.TRACE);
- testLogger.clearOutput();
- watcherLogger.clearOutput();
-
- // When both of the pending event are executed successfully by DataNode
- CommandStatus cmdStatus1 = CommandStatus.newBuilder()
- .setCmdId(id1)
- .setStatus(CommandStatus.Status.EXECUTED)
- .setType(Type.closeContainerCommand).build();
- CommandStatus cmdStatus2 = CommandStatus.newBuilder()
- .setCmdId(id2)
- .setStatus(CommandStatus.Status.EXECUTED)
- .setType(Type.closeContainerCommand).build();
- // File events to watcher
- testLogger.clearOutput();
- watcherLogger.clearOutput();
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id1)));
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id2)));
- Thread.sleep(10L);
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_STATUS,
- new CloseContainerStatus(cmdStatus1));
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_STATUS,
- new CloseContainerStatus(cmdStatus2));
-
- Thread.sleep(1000*3L);
- // validation
- assertTrue(watcherLogger.getOutput().contains("CloseContainerCommand "
- + "for containerId: " + id1 + " executed"));
- assertTrue(watcherLogger.getOutput().contains("CloseContainerCommand "
- + "for containerId: " + id2 + " executed"));
- assertFalse(testLogger.getOutput().contains("Handling "
- + "closeContainerEvent for containerId: id=" + id1));
- assertFalse(testLogger.getOutput().contains("Handling "
- + "closeContainerEvent for containerId: id=" + id2));
- }
-
- private void setupWatcher(long time) {
- leaseManager = new LeaseManager<>("TestCloseContainerWatcher#LeaseManager",
- time);
- leaseManager.start();
- watcher = new CloseContainerWatcher(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- SCMEvents.CLOSE_CONTAINER_STATUS, leaseManager, containerManager);
- queue = new EventQueue();
- watcher.start(queue);
- }
-
- /*
- * This test will fire two retryable closeContainer events. Both will timeout.
- * First event container will be open at time of handling so it should be
- * sent back to appropriate handler. Second event container will be closed,
- * so it should not be retried.
- * */
- @Test
- public void testWatcherRetryableTimeoutHandling() throws InterruptedException,
- IOException {
-
- long id1 = HddsIdFactory.getLongId();
- long id2 = HddsIdFactory.getLongId();
- setupWatcher(1000L);
- queue.addHandler(SCMEvents.CLOSE_CONTAINER, this);
- setupMock(id1, id2, false);
- GenericTestUtils.LogCapturer testLogger = GenericTestUtils.LogCapturer
- .captureLogs(LOG);
- testLogger.clearOutput();
-
- // File events to watcher
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id1)));
- queue.fireEvent(SCMEvents.CLOSE_CONTAINER_RETRYABLE_REQ,
- new CloseContainerRetryableReq(ContainerID.valueof(id2)));
-
- Thread.sleep(1000L + 10);
-
- // validation
- assertTrue(testLogger.getOutput().contains("Handling "
- + "closeContainerEvent for containerId: id=" + id1));
- assertFalse(testLogger.getOutput().contains("Handling "
- + "closeContainerEvent for containerId: id=" + id2));
- }
-
-
- private void setupMock(long id1, long id2, boolean isOpen)
- throws IOException {
- ContainerInfo containerInfo = Mockito.mock(ContainerInfo.class);
- ContainerInfo containerInfo2 = Mockito.mock(ContainerInfo.class);
- when(containerManager.getContainer(ContainerID.valueof(id1)))
- .thenReturn(containerInfo);
- when(containerManager.getContainer(ContainerID.valueof(id2)))
- .thenReturn(containerInfo2);
- when(containerInfo.isOpen()).thenReturn(true);
- when(containerInfo2.isOpen()).thenReturn(isOpen);
- }
-
- @Override
- public void onMessage(ContainerID containerID, EventPublisher publisher) {
- LOG.info("Handling closeContainerEvent for containerId: {}", containerID);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index 3a2ce78..a089621 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -413,17 +413,13 @@ public class TestEndPoint {
serverAddress, 3000);
Map<Long, CommandStatus> map = stateContext.getCommandStatusMap();
assertNotNull(map);
- assertEquals("Should have 3 objects", 3, map.size());
- assertTrue(map.containsKey(Long.valueOf(1)));
+ assertEquals("Should have 2 objects", 2, map.size());
assertTrue(map.containsKey(Long.valueOf(2)));
assertTrue(map.containsKey(Long.valueOf(3)));
- assertTrue(map.get(Long.valueOf(1)).getType()
- .equals(Type.closeContainerCommand));
assertTrue(map.get(Long.valueOf(2)).getType()
.equals(Type.replicateContainerCommand));
assertTrue(
map.get(Long.valueOf(3)).getType().equals(Type.deleteBlocksCommand));
- assertTrue(map.get(Long.valueOf(1)).getStatus().equals(Status.PENDING));
assertTrue(map.get(Long.valueOf(2)).getStatus().equals(Status.PENDING));
assertTrue(map.get(Long.valueOf(3)).getStatus().equals(Status.PENDING));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index fbc3420..2d786c5 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.CommandQueue;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
@@ -34,7 +33,6 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -76,19 +74,6 @@ public class ReplicationNodeManagerMock implements NodeManager {
}
/**
- * Removes a data node from the management of this Node Manager.
- *
- * @param node - DataNode.
- * @throws NodeNotFoundException
- */
- @Override
- public void removeNode(DatanodeDetails node)
- throws NodeNotFoundException {
- nodeStateMap.remove(node);
-
- }
-
- /**
* Gets all Live Datanodes that is currently communicating with SCM.
*
* @param nodestate - State of the node
@@ -170,7 +155,7 @@ public class ReplicationNodeManagerMock implements NodeManager {
* @return Set of PipelineID
*/
@Override
- public Set<PipelineID> getPipelineByDnID(UUID dnId) {
+ public Set<PipelineID> getPipelines(DatanodeDetails dnId) {
throw new UnsupportedOperationException("Not yet implemented");
}
@@ -196,24 +181,12 @@ public class ReplicationNodeManagerMock implements NodeManager {
* Update set of containers available on a datanode.
* @param uuid - DatanodeID
* @param containerIds - Set of containerIDs
- * @throws SCMException - if datanode is not known. For new datanode use
- * addDatanodeInContainerMap call.
+ * @throws NodeNotFoundException - if datanode is not known. For new datanode
+ * use addDatanodeInContainerMap call.
*/
@Override
- public void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
- throws SCMException {
- throw new UnsupportedOperationException("Not yet implemented");
- }
-
- /**
- * Process containerReport received from datanode.
- * @param uuid - DataonodeID
- * @param containerIds - Set of containerIDs
- * @return The result after processing containerReport
- */
- @Override
- public ReportResult<ContainerID> processContainerReport(UUID uuid,
- Set<ContainerID> containerIds) {
+ public void setContainers(DatanodeDetails uuid, Set<ContainerID> containerIds)
+ throws NodeNotFoundException {
throw new UnsupportedOperationException("Not yet implemented");
}
@@ -223,20 +196,7 @@ public class ReplicationNodeManagerMock implements NodeManager {
* @return - set of containerIDs
*/
@Override
- public Set<ContainerID> getContainers(UUID uuid) {
- throw new UnsupportedOperationException("Not yet implemented");
- }
-
- /**
- * Insert a new datanode with set of containerIDs for containers available
- * on it.
- * @param uuid - DatanodeID
- * @param containerIDs - Set of ContainerIDs
- * @throws SCMException - if datanode already exists
- */
- @Override
- public void addDatanodeInContainerMap(UUID uuid,
- Set<ContainerID> containerIDs) throws SCMException {
+ public Set<ContainerID> getContainers(DatanodeDetails uuid) {
throw new UnsupportedOperationException("Not yet implemented");
}
@@ -329,7 +289,8 @@ public class ReplicationNodeManagerMock implements NodeManager {
* @param nodeReport
*/
@Override
- public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) {
+ public void processNodeReport(DatanodeDetails dnUuid,
+ NodeReportProto nodeReport) {
// do nothing.
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
index 03c99ef..143c4e3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
@@ -21,14 +21,14 @@ import java.util.UUID;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.states.ContainerStateMap;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -39,8 +39,6 @@ import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.TimeoutException;
-import org.slf4j.event.Level;
-
/**
* Tests for ContainerStateManager.
*/
@@ -317,9 +315,6 @@ public class TestContainerStateManagerIntegration {
@Test
public void testReplicaMap() throws Exception {
- GenericTestUtils.setLogLevel(ContainerStateMap.getLOG(), Level.DEBUG);
- GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
- .captureLogs(ContainerStateMap.getLOG());
DatanodeDetails dn1 = DatanodeDetails.newBuilder().setHostName("host1")
.setIpAddress("1.1.1.1")
.setUuid(UUID.randomUUID().toString()).build();
@@ -347,10 +342,12 @@ public class TestContainerStateManagerIntegration {
// Test 2: Add replica nodes and then test
ContainerReplica replicaOne = ContainerReplica.newBuilder()
.setContainerID(id)
+ .setContainerState(ContainerReplicaProto.State.OPEN)
.setDatanodeDetails(dn1)
.build();
ContainerReplica replicaTwo = ContainerReplica.newBuilder()
.setContainerID(id)
+ .setContainerState(ContainerReplicaProto.State.OPEN)
.setDatanodeDetails(dn2)
.build();
containerStateManager.updateContainerReplica(id, replicaOne);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
index 74cbea4..88b5f7f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
@@ -93,7 +93,7 @@ public class TestNode2PipelineMap {
// get pipeline details by dnid
Set<PipelineID> pipelines = scm.getScmNodeManager()
- .getPipelineByDnID(dns.get(0).getUuid());
+ .getPipelines(dns.get(0));
Assert.assertEquals(1, pipelines.size());
pipelines.forEach(p -> Assert.assertEquals(p,
ratisContainer.getPipeline().getId()));
@@ -116,7 +116,7 @@ public class TestNode2PipelineMap {
pipelineManager.finalizePipeline(ratisContainer.getPipeline().getId());
pipelineManager.removePipeline(ratisContainer.getPipeline().getId());
pipelines = scm.getScmNodeManager()
- .getPipelineByDnID(dns.get(0).getUuid());
+ .getPipelines(dns.get(0));
Assert.assertEquals(0, pipelines.size());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
index 42d3063..6121a65 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
@@ -115,8 +115,8 @@ public class TestPipelineClose {
pipelineManager.removePipeline(pipeline1.getId());
for (DatanodeDetails dn : ratisContainer1.getPipeline().getNodes()) {
// Assert that the pipeline has been removed from Node2PipelineMap as well
- Assert.assertEquals(scm.getScmNodeManager().getPipelineByDnID(
- dn.getUuid()).size(), 0);
+ Assert.assertEquals(scm.getScmNodeManager().getPipelines(
+ dn).size(), 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c80f753b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index c49a98b..5f57e7b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -243,8 +243,10 @@ public class TestBlockDeletion {
ContainerReportsProto dummyReport = dummyReportsBuilder.build();
logCapturer.clearOutput();
- scm.getContainerManager().processContainerReports(
- cluster.getHddsDatanodes().get(0).getDatanodeDetails(), dummyReport);
+ cluster.getHddsDatanodes().get(0)
+ .getDatanodeStateMachine().getContext().addReport(dummyReport);
+ cluster.getHddsDatanodes().get(0)
+ .getDatanodeStateMachine().triggerHeartbeat();
// wait for event to be handled by event handler
Thread.sleep(1000);
String output = logCapturer.getOutput();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org