You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/10/24 00:40:53 UTC
[01/50] hadoop git commit: HADOOP-14935. Azure: POSIX permissions are
taking effect in access() method even when authorization is enabled.
Contributed by Santhosh G Nayak
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 0fb394b79 -> 2628d1b06
HADOOP-14935. Azure: POSIX permissions are taking effect in access() method even when authorization is enabled.
Contributed by Santhosh G Nayak
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fcc3a1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fcc3a1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fcc3a1f
Branch: refs/heads/HDFS-7240
Commit: 9fcc3a1fc8cab873034f5c308ceb2d5671a954e8
Parents: 20575ec
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 16 16:01:47 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 16 16:01:47 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 63 +++--
.../hadoop/fs/azure/security/Constants.java | 1 +
.../TestNativeAzureFileSystemAuthorization.java | 280 ++++++++++++++-----
3 files changed, 254 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fcc3a1f/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 9effb3b..85a46ea 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -71,9 +71,11 @@ import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem;
import org.apache.hadoop.fs.azure.security.Constants;
import org.apache.hadoop.fs.azure.security.RemoteWasbDelegationTokenManager;
import org.apache.hadoop.fs.azure.security.WasbDelegationTokenManager;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
@@ -2650,22 +2652,6 @@ public class NativeAzureFileSystem extends FileSystem {
public FileStatus getFileStatus(Path f) throws FileNotFoundException, IOException {
LOG.debug("Getting the file status for {}", f.toString());
-
- // Capture the absolute path and the path to key.
- Path absolutePath = makeAbsolute(f);
-
- if (!isRenamePendingFile(absolutePath)) {
- Path ancestor = getAncestor(absolutePath);
- if (ancestor.equals(absolutePath) && !ancestor.equals(new Path("/"))) {
- performAuthCheck(ancestor.getParent(), WasbAuthorizationOperations.READ,
- "getFileStatus", absolutePath);
- }
- else {
- performAuthCheck(ancestor, WasbAuthorizationOperations.READ,
- "getFileStatus", absolutePath);
- }
- }
-
return getFileStatusInternal(f);
}
@@ -2693,7 +2679,15 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
- protected FileStatus getFileStatusInternal(Path f) throws FileNotFoundException, IOException {
+ /**
+ * Inner implementation of {@link #getFileStatus(Path)}.
+ * Return a file status object that represents the path.
+ * @param f The path we want information from
+ * @return a FileStatus object
+ * @throws FileNotFoundException when the path does not exist
+ * @throws IOException Other failure
+ */
+ private FileStatus getFileStatusInternal(Path f) throws FileNotFoundException, IOException {
Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
@@ -3707,6 +3701,41 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
+ @Override
+ public void access(Path path, FsAction mode) throws IOException {
+ if (azureAuthorization && authorizer != null) {
+ try {
+ // Required to check the existence of the path.
+ getFileStatus(path);
+ switch (mode) {
+ case READ:
+ case READ_EXECUTE:
+ performAuthCheck(path, WasbAuthorizationOperations.READ, "access", path);
+ break;
+ case WRITE:
+ case WRITE_EXECUTE:
+ performAuthCheck(path, WasbAuthorizationOperations.WRITE, "access",
+ path);
+ break;
+ case READ_WRITE:
+ case ALL:
+ performAuthCheck(path, WasbAuthorizationOperations.READ, "access", path);
+ performAuthCheck(path, WasbAuthorizationOperations.WRITE, "access",
+ path);
+ break;
+ case EXECUTE:
+ case NONE:
+ default:
+ break;
+ }
+ } catch (WasbAuthorizationException wae){
+ throw new AccessControlException(wae);
+ }
+ } else {
+ super.access(path, mode);
+ }
+ }
+
/**
* A handler that defines what to do with blobs whose upload was
* interrupted.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fcc3a1f/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
index fa63837..792fe0a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java
@@ -34,6 +34,7 @@ public final class Constants {
*/
public static final String AZURE_ENABLE_SPNEGO_TOKEN_CACHE =
"fs.azure.enable.spnego.token.cache";
+
/**
* Parameter to be used for impersonation.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fcc3a1f/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
index c8d6507..4c61827 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -31,9 +32,12 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Assume;
@@ -145,7 +149,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -171,7 +174,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -199,7 +201,6 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("create", testPath);
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -225,7 +226,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -331,7 +331,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
/* for rename */
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -362,7 +361,6 @@ public class TestNativeAzureFileSystemAuthorization
/* to create parent dir */
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, false);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -395,8 +393,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dir */
authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, false);
- authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -426,8 +422,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dirs */
authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, true);
- authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -605,9 +599,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, true);
/* Required for asserPathExists calls */
- authorizer.addAuthRuleForOwner("/", READ, true);
- authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -648,9 +639,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(parentSrcDir.toString(),
WRITE, true);
/* Required for asserPathExists calls */
- authorizer.addAuthRuleForOwner("/", READ, true);
- authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -672,10 +660,6 @@ public class TestNativeAzureFileSystemAuthorization
WRITE, getCurrentUserShortName(), true);
authorizer.addAuthRule(parentDstDir.toString(),
WRITE, getCurrentUserShortName(), true);
- authorizer.addAuthRule(parentSrcDir.toString(),
- READ, getCurrentUserShortName(), true);
- authorizer.addAuthRule(parentDstDir.toString(),
- READ, getCurrentUserShortName(), true);
try {
fs.rename(srcPath, dstPath);
@@ -711,9 +695,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(parentSrcDir.toString(),
WRITE, true);
/* Required for asserPathExists calls */
- authorizer.addAuthRuleForOwner("/", READ, true);
- authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -735,10 +716,6 @@ public class TestNativeAzureFileSystemAuthorization
WRITE, getCurrentUserShortName(), true);
authorizer.addAuthRule(parentDstDir.toString(),
WRITE, getCurrentUserShortName(), true);
- authorizer.addAuthRule(parentSrcDir.toString(),
- READ, getCurrentUserShortName(), true);
- authorizer.addAuthRule(parentDstDir.toString(),
- READ, getCurrentUserShortName(), true);
// should return false since srcPath does not exist.
assertRenameOutcome(fs, srcPath, dstPath, false);
assertPathDoesNotExist(fs, "destPath exists", dstPath);
@@ -840,7 +817,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -865,7 +841,6 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("delete", testPath);
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -875,7 +850,6 @@ public class TestNativeAzureFileSystemAuthorization
/* Remove permissions for delete to force failure */
authorizer.deleteAllAuthRules();
authorizer.addAuthRuleForOwner("/", WRITE, false);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -884,7 +858,6 @@ public class TestNativeAzureFileSystemAuthorization
/* Restore permissions to force a successful delete */
authorizer.deleteAllAuthRules();
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -905,8 +878,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(childPath, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true); // for create and delete
- authorizer.addAuthRuleForOwner(childPath.toString(), READ, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
authorizer.addAuthRuleForOwner("/testDeleteIntermediateFolder*",
WRITE, true); // for recursive delete
fs.updateWasbAuthorizer(authorizer);
@@ -939,8 +910,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted*",
WRITE, true);
- authorizer.addAuthRuleForOwner(childPath1.toString(), READ, true);
- authorizer.addAuthRuleForOwner(childPath2.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -955,9 +924,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
authorizer.addAuthRuleForOwner(childPath2.toString(), WRITE, true);
authorizer.addAuthRuleForOwner(childPath1.toString(), WRITE, false);
- authorizer.addAuthRuleForOwner(childPath1.toString(), READ, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
assertFalse(fs.delete(parentDir, true));
@@ -986,7 +952,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1023,7 +988,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1041,8 +1005,6 @@ public class TestNativeAzureFileSystemAuthorization
try {
authorizer.addAuthRule(parentDir.toString(), WRITE,
getCurrentUserShortName(), true);
- authorizer.addAuthRule(parentDir.toString(), READ,
- getCurrentUserShortName(), true);
fs.delete(testPath, true);
return null;
}
@@ -1075,8 +1037,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testRecursiveDeleteSucceedsWithStickybit*",
WRITE, true);
- authorizer.addAuthRuleForOwner(childDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1113,8 +1073,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testRecursiveDeleteFailsWithStickybit*",
WRITE, true);
- authorizer.addAuthRuleForOwner(childDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1170,7 +1128,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(
"/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1185,11 +1142,8 @@ public class TestNativeAzureFileSystemAuthorization
@Override
public Void run() throws Exception {
authorizer.addAuthRule("/", WRITE, getCurrentUserShortName(), true);
- authorizer.addAuthRule(parentDir.toString(), READ, getCurrentUserShortName(), true);
- authorizer.addAuthRule(testFolderPath.toString(), READ, getCurrentUserShortName(), true);
authorizer.addAuthRule("/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
WRITE, getCurrentUserShortName(), true);
- authorizer.addAuthRule("/", READ, getCurrentUserShortName(), true);
fs.create(testFolderPath); // the folder will have owner as dummyuser
ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
@@ -1228,8 +1182,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(
"/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit*",
WRITE, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1244,7 +1196,6 @@ public class TestNativeAzureFileSystemAuthorization
@Override
public Void run() throws Exception {
authorizer.addAuthRule(parentDir.toString(), WRITE, getCurrentUserShortName(), true);
- authorizer.addAuthRule(parentDir.toString(), READ, getCurrentUserShortName(), true);
fs.create(testFilePath);
ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
@@ -1282,10 +1233,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testDeleteScenarioForRoot*",
WRITE, true);
- authorizer.addAuthRuleForOwner(childPath1.toString(), READ, true);
- authorizer.addAuthRuleForOwner(childPath2.toString(), READ, true);
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1329,7 +1276,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsAccessCheckPositive/1/2/3");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1352,7 +1298,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsWithExistingHierarchyCheckPositive1");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1386,10 +1331,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(childPath1.toString(),
WRITE, true);
- authorizer.addAuthRuleForOwner(childPath1.getParent().toString(), READ, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
- authorizer.addAuthRuleForOwner(childPath3.getParent().toString(), READ, true);
-
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1464,8 +1405,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
// additional rule used for assertPathExists
- authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1543,7 +1482,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerNegative");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
final String owner;
@@ -1582,7 +1520,6 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerPositive");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
String newOwner = "user2";
@@ -1626,7 +1563,6 @@ public class TestNativeAzureFileSystemAuthorization
Configuration conf = fs.getConf();
authorizer.init(conf);
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
final String newOwner = "newowner";
@@ -1670,7 +1606,6 @@ public class TestNativeAzureFileSystemAuthorization
Configuration conf = fs.getConf();
authorizer.init(conf);
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
UserGroupInformation user = UserGroupInformation.createUserForTesting(
@@ -1714,8 +1649,6 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
/* Remove nextline after fixing createInternal from FolderRenamePending */
authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
- authorizer.addAuthRuleForOwner(srcPath.getParent().toString(), READ, true);
- authorizer.addAuthRuleForOwner(dstFilePath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1834,6 +1767,197 @@ public class TestNativeAzureFileSystemAuthorization
}
/**
+ * Test access when requested permissions match the existing permissions.
+ */
+ @Test
+ public void testAccessWhenPermissionsMatchForAllAndReadWrite() throws Throwable {
+ Configuration conf = fs.getConf();
+ fs.setConf(conf);
+ final Path testPath = new Path("/testAccessWhenPermissionsMatchForAllAndReadWrite");
+
+ // For All and Read-Write FsAction.
+ authorizer.init(conf);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+ try {
+ fs.mkdirs(testPath);
+ assertPathExists(fs, "test path does not exist", testPath);
+ fs.access(testPath, FsAction.ALL);
+ fs.access(testPath, FsAction.READ_WRITE);
+ } finally {
+ recursiveDelete(testPath);
+ }
+
+ }
+
+ /**
+ * Test access when Write and Write-Execute match the existing permissions.
+ * @throws Throwable
+ */
+ @Test
+ public void testAccessWhenPermissionsMatchForWriteAndWriteExecute() throws Throwable {
+ Configuration conf = fs.getConf();
+ fs.setConf(conf);
+ final Path testPath = new Path("/testAccessWhenPermissionsMatchForWriteAndWriteExecute");
+ // For Write and Write-Execute FsAction.
+ authorizer.init(conf);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+ try {
+ fs.mkdirs(testPath);
+ assertPathExists(fs, "test path does not exist", testPath);
+ fs.access(testPath, FsAction.WRITE);
+ fs.access(testPath, FsAction.WRITE_EXECUTE);
+ } finally {
+ recursiveDelete(testPath);
+ }
+ }
+
+ /**
+ * Test access when Read and Read-Execute match the existing permissions.
+ * @throws Throwable
+ */
+ @Test
+ public void testAccessWhenPermissionsMatchForReadAndReadExecute() throws Throwable {
+ Configuration conf = fs.getConf();
+ fs.setConf(conf);
+ final Path testPath = new Path("/testAccessWhenPermissionsMatchForReadAndReadExecute");
+ // For Read and Read-Execute FsAction.
+ authorizer.init(conf);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+ try {
+ fs.mkdirs(testPath);
+ assertPathExists(fs, "test path does not exist", testPath);
+ fs.access(testPath, FsAction.READ);
+ fs.access(testPath, FsAction.READ_EXECUTE);
+ } finally {
+ recursiveDelete(testPath);
+ }
+ }
+
+ /**
+ * Test access when Execute and None match the existing permissions.
+ * @throws Throwable
+ */
+ @Test
+ public void testAccessWhenPermissionsMatchForExecuteAndNone() throws Throwable {
+ Configuration conf = fs.getConf();
+ fs.setConf(conf);
+ final Path testPath = new Path("/testAccessWhenPermissionsMatchForExecuteAndNone");
+ // For Execute and None FsAction.
+ authorizer.init(conf);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+ try {
+ fs.mkdirs(testPath);
+ assertPathExists(fs, "test path does not exist", testPath);
+ fs.access(testPath, FsAction.EXECUTE);
+ fs.access(testPath, FsAction.NONE);
+ } finally {
+ recursiveDelete(testPath);
+ }
+ }
+
+ /**
+ * Test access when requested permissions do not match existing permissions.
+ */
+ @Test
+ public void testAccessWhenPermissionsDoNotMatch() throws Throwable{
+ Configuration conf = fs.getConf();
+ fs.setConf(conf);
+ final Path testPath = new Path("/testAccessWhenPermissionsDoNotMatch");
+
+ authorizer.init(conf);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+ try {
+ fs.mkdirs(testPath);
+ assertPathExists(fs, "test path does not exist", testPath);
+ assertNoAccess(testPath, FsAction.ALL);
+ assertNoAccess(testPath, FsAction.WRITE);
+ assertNoAccess(testPath, FsAction.WRITE_EXECUTE);
+ } finally {
+ recursiveDelete(testPath);
+ }
+ }
+
+ /**
+ * Test access when file does not exist and permissions match.
+ */
+ @Test
+ public void testAccessFileDoesNotExist() throws Throwable{
+ expectedEx.expect(FileNotFoundException.class);
+ Configuration conf = fs.getConf();
+ fs.setConf(conf);
+ final Path testPath = new Path("/testAccessFileDoesNotExist");
+
+ authorizer.init(conf);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+ assertPathDoesNotExist(fs, "test path exists", testPath);
+ fs.access(testPath, FsAction.ALL);
+ }
+
+ /**
+ * Test access when file does not exist and permissions do not match.
+ */
+ @Test
+ public void testAccessFileDoesNotExistWhenNoAccessPermission() throws Throwable {
+ expectedEx.expect(FileNotFoundException.class);
+ Configuration conf = fs.getConf();
+ fs.setConf(conf);
+ final Path testPath = new Path("/testAccessFileDoesNotExistWhenNoAccessPermission");
+
+ authorizer.init(conf);
+ fs.updateWasbAuthorizer(authorizer);
+ assertPathDoesNotExist(fs, "test path exists", testPath);
+ fs.access(testPath, FsAction.ALL);
+ }
+
+ /**
+ * Test access for file and intermediate directory after creating
+ * file with intermediate directory.
+ */
+ @Test
+ public void testAccessForFileAndIntermediateDirectoryCreated() throws Throwable {
+ Path parentDir = new Path("/testAccessDirectory");
+ Path intermediateDir = new Path(parentDir, "intermediateDir");
+ Path testPath = new Path(intermediateDir, "test.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+
+ // Recursive owner Rule is to determine the permission on intermediate dirs.
+ authorizer.addAuthRuleForOwner(parentDir.toString()+"/*", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString()+"/*", READ, true);
+
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.mkdirs(parentDir);
+ fs.create(testPath);
+ assertPathExists(fs, "testPath was not created", testPath);
+ fs.access(parentDir, FsAction.WRITE);
+ fs.access(parentDir, FsAction.WRITE_EXECUTE);
+
+ fs.access(intermediateDir, FsAction.ALL);
+ fs.access(intermediateDir, FsAction.READ_WRITE);
+
+ fs.access(testPath, FsAction.ALL);
+ fs.access(testPath, FsAction.READ_WRITE);
+ } finally {
+ recursiveDelete(testPath);
+ }
+ }
+
+ /**
* Helper method to test setPermission scenarios. This method handles both positive
* and negative scenarios of setPermission tests
*/
@@ -1849,7 +1973,6 @@ public class TestNativeAzureFileSystemAuthorization
final Path testPath = new Path(path);
final FsPermission newPermission = new FsPermission(FULL_PERMISSION_WITH_STICKYBIT);
authorizer.addAuthRule("/", WRITE, getCurrentUserShortName(), true);
- authorizer.addAuthRule("/", READ, getCurrentUserShortName(), true);
fs.updateWasbAuthorizer(authorizer);
if (chmodAllowedUsers != null && !chmodAllowedUsers.isEmpty()) {
@@ -1895,7 +2018,6 @@ public class TestNativeAzureFileSystemAuthorization
public Void run() throws Exception {
try {
//READ access required for getFileStatus
- authorizer.addAuthRule("/", READ, getCurrentUserShortName(), true);
fs.setPermission(testPath, newPermission);
fail("Failing test because setPermission was expected to throw");
@@ -1961,4 +2083,16 @@ public class TestNativeAzureFileSystemAuthorization
assertEquals("Wrong owner in " + status, owner, status.getOwner());
}
+ private void assertNoAccess(final Path path, final FsAction action)
+ throws Exception {
+ LambdaTestUtils.intercept(AccessControlException.class,
+ new Callable<String>() {
+ @Override
+ public String call() throws Exception {
+ fs.access(path, action);
+ return "Access granted to " + path + " for action " + action;
+ }
+ }
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] hadoop git commit: HADOOP-14942. DistCp#cleanup() should
check whether jobFS is null. Contributed by Andras Bokor.
Posted by ae...@apache.org.
HADOOP-14942. DistCp#cleanup() should check whether jobFS is null.
Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f36cbc84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f36cbc84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f36cbc84
Branch: refs/heads/HDFS-7240
Commit: f36cbc847560d53e7955ced9ce7ce2773c805793
Parents: 480187a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 20 22:27:04 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 20 22:27:04 2017 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/tools/DistCp.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36cbc84/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index df9c328..909494b 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -453,10 +453,12 @@ public class DistCp extends Configured implements Tool {
private synchronized void cleanup() {
try {
- if (metaFolder == null) return;
-
- jobFS.delete(metaFolder, true);
- metaFolder = null;
+ if (metaFolder != null) {
+ if (jobFS != null) {
+ jobFS.delete(metaFolder, true);
+ }
+ metaFolder = null;
+ }
} catch (IOException e) {
LOG.error("Unable to cleanup meta folder: " + metaFolder, e);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] hadoop git commit: YARN-7353. Improved volume mount check for
directories and unit test compatibility on RHEL7. Contributed by Eric Badger.
Posted by ae...@apache.org.
YARN-7353. Improved volume mount check for directories and unit test compatibility on RHEL7. Contributed by Eric Badger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b61144a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b61144a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b61144a9
Branch: refs/heads/HDFS-7240
Commit: b61144a93d9306624378a93944d0a08c60436554
Parents: 1f4cdf1
Author: Eric Yang <ey...@apache.org>
Authored: Fri Oct 20 12:02:06 2017 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Oct 20 12:02:06 2017 -0400
----------------------------------------------------------------------
.../container-executor/impl/utils/docker-util.c | 5 +-
.../test/utils/test_docker_util.cc | 103 ++++++++++---------
2 files changed, 55 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61144a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 860320d..e8e2b9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -687,8 +687,9 @@ static int check_mount_permitted(const char **permitted_mounts, const char *requ
}
// directory check
permitted_mount_len = strlen(permitted_mounts[i]);
- if (permitted_mount_len > 0
- && permitted_mounts[i][permitted_mount_len - 1] == '/') {
+ struct stat path_stat;
+ stat(permitted_mounts[i], &path_stat);
+ if(S_ISDIR(path_stat.st_mode)) {
if (strncmp(normalized_path, permitted_mounts[i], permitted_mount_len) == 0) {
ret = 1;
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61144a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index c627ca8..c42cd78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -429,13 +429,14 @@ namespace ContainerExecutor {
}
TEST_F(TestDockerUtil, test_check_mount_permitted) {
- const char *permitted_mounts[] = {"/usr/", "/bin/ls", NULL};
+ const char *permitted_mounts[] = {"/etc", "/usr/bin/touch", "/tmp/", NULL};
std::vector<std::pair<std::string, int> > test_data;
- test_data.push_back(std::make_pair<std::string, int>("/usr", 1));
- test_data.push_back(std::make_pair<std::string, int>("/usr/", 1));
- test_data.push_back(std::make_pair<std::string, int>("/bin/ls", 1));
- test_data.push_back(std::make_pair<std::string, int>("//bin/", 0));
- test_data.push_back(std::make_pair<std::string, int>("/tmp/random-file", -1));
+ test_data.push_back(std::make_pair<std::string, int>("/etc", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/etc/", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/etc/passwd", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/usr/bin/touch", 1));
+ test_data.push_back(std::make_pair<std::string, int>("//usr/", 0));
+ test_data.push_back(std::make_pair<std::string, int>("/etc/random-file", -1));
std::vector<std::pair<std::string, int> >::const_iterator itr;
for (itr = test_data.begin(); itr != test_data.end(); ++itr) {
@@ -446,8 +447,8 @@ namespace ContainerExecutor {
TEST_F(TestDockerUtil, test_normalize_mounts) {
const int entries = 4;
- const char *permitted_mounts[] = {"/home", "/usr", "/bin/ls", NULL};
- const char *expected[] = {"/home/", "/usr/", "/bin/ls", NULL};
+ const char *permitted_mounts[] = {"/home", "/etc", "/usr/bin/touch", NULL};
+ const char *expected[] = {"/home/", "/etc/", "/usr/bin/touch", NULL};
char **ptr = static_cast<char **>(malloc(entries * sizeof(char *)));
for (int i = 0; i < entries; ++i) {
if (permitted_mounts[i] != NULL) {
@@ -659,22 +660,19 @@ namespace ContainerExecutor {
const int buff_len = 1024;
char buff[buff_len];
int ret = 0;
- std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls,..\n "
- "docker.allowed.ro-mounts=/bin/cat";
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/opt,/var,/usr/bin/touch,..\n "
+ "docker.allowed.ro-mounts=/etc/passwd";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n rw-mounts=/var:/var", "-v '/var:/var' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n rw-mounts=/var/:/var/", "-v '/var/:/var/' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr:/usr", "-v '/usr:/usr' "));
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin/touch:/usr/bin/touch",
+ "-v '/usr/bin/touch:/usr/bin/touch' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/:/usr", "-v '/usr/:/usr' "));
- file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls' "));
- file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin:/mydisk1,/var/log/:/mydisk2",
- "-v '/usr/bin:/mydisk1' -v '/var/log/:/mydisk2' "));
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
+ "-v '/opt:/mydisk1' -v '/var/log/:/mydisk2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n", ""));
write_container_executor_cfg(container_executor_cfg_contents);
@@ -705,10 +703,10 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n rw-mounts=/home:/home",
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/lib:/lib",
static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/cat:/bin/cat",
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin/:/usr/bin",
static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n docker-command=run\n rw-mounts=/blah:/blah",
@@ -768,27 +766,30 @@ namespace ContainerExecutor {
const int buff_len = 1024;
char buff[buff_len];
int ret = 0;
- std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls\n "
- "docker.allowed.ro-mounts=/bin/cat,/bin/ln";
+
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/home/,/var,/usr/bin/touch,..\n "
+ "docker.allowed.ro-mounts=/etc/passwd,/etc/group";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n ro-mounts=/var:/var", "-v '/var:/var:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr:/usr", "-v '/usr:/usr:ro' "));
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home", "-v '/home:/home:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/:/usr", "-v '/usr/:/usr:ro' "));
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/home/:/home", "-v '/home/:/home:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls:ro' "));
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/bin/touch:/usr/bin/touch",
+ "-v '/usr/bin/touch:/usr/bin/touch:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ln:/bin/ln", "-v '/bin/ln:/bin/ln:ro' "));
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc/group:/etc/group",
+ "-v '/etc/group:/etc/group:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/cat:/bin/cat",
- "-v '/bin/cat:/bin/cat:ro' "));
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc/passwd:/etc/passwd",
+ "-v '/etc/passwd:/etc/passwd:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/bin:/mydisk1,/bin/cat:/bin/cat",
- "-v '/usr/bin:/mydisk1:ro' -v '/bin/cat:/bin/cat:ro' "));
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd",
+ "-v '/var/log:/mydisk1:ro' -v '/etc/passwd:/etc/passwd:ro' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n", ""));
write_container_executor_cfg(container_executor_cfg_contents);
@@ -819,7 +820,7 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home",
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc:/etc",
static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n docker-command=run\n ro-mounts=/blah:/blah",
@@ -855,7 +856,7 @@ namespace ContainerExecutor {
TEST_F(TestDockerUtil, test_docker_run_privileged) {
- std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/touch\n"
" docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
" docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n"
" docker.allowed.devices=/dev/test";
@@ -882,36 +883,36 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
- " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " -v '/usr/bin/touch:/usr/bin/touch:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
" --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
"'test_script.sh' 'arg1' 'arg2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
- " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ " -v '/usr/bin/touch:/usr/bin/touch:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
"--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
" 'test_script.sh' 'arg1' 'arg2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
- " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ " -v '/usr/bin/touch:/usr/bin/touch:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
"--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' "
"'bash' 'test_script.sh' 'arg1' 'arg2' "));
@@ -919,12 +920,12 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
- " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ " -v '/usr/bin/touch:/usr/bin/touch:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
"--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' "
"--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
@@ -945,7 +946,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/var/log:/var/log\n"
+ " ro-mounts=/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/var/log:/var/log\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -955,7 +956,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/bin:/bin,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/bin:/bin,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -965,7 +966,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -975,7 +976,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -985,7 +986,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -996,11 +997,11 @@ namespace ContainerExecutor {
TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
- std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/touch\n"
" docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
" docker.allowed.capabilities=CHOWN,SETUID\n"
" docker.allowed.devices=/dev/test",
- "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ "[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/touch\n"
" docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
" docker.allowed.capabilities=CHOWN,SETUID\n"
" privileged=0\n"
@@ -1029,24 +1030,24 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
- " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " -v '/usr/bin/touch:/usr/bin/touch:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
" --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
"'test_script.sh' 'arg1' 'arg2' "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
- " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ " -v '/usr/bin/touch:/usr/bin/touch:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
"--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
" 'test_script.sh' 'arg1' 'arg2' "));
@@ -1054,7 +1055,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/touch:/usr/bin/touch\n rw-mounts=/tmp:/tmp\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] hadoop git commit: HDFS-12518. Re-encryption should handle
task cancellation and progress better.
Posted by ae...@apache.org.
HDFS-12518. Re-encryption should handle task cancellation and progress better.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/248d9b6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/248d9b6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/248d9b6f
Branch: refs/heads/HDFS-7240
Commit: 248d9b6fff648cdb02581d458556b6f7c090ef1a
Parents: 14b3c26
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Oct 20 16:24:33 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Oct 20 16:25:04 2017 -0700
----------------------------------------------------------------------
.../server/namenode/FSDirEncryptionZoneOp.java | 8 +-
.../server/namenode/ReencryptionHandler.java | 106 ++++++++++++-------
.../server/namenode/ReencryptionUpdater.java | 29 +++--
.../hdfs/server/namenode/TestReencryption.java | 59 ++++++++++-
4 files changed, 150 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/248d9b6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index e284b15..7dcb8ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -380,10 +380,16 @@ final class FSDirEncryptionZoneOp {
static void saveFileXAttrsForBatch(FSDirectory fsd,
List<FileEdekInfo> batch) {
assert fsd.getFSNamesystem().hasWriteLock();
+ assert !fsd.hasWriteLock();
if (batch != null && !batch.isEmpty()) {
for (FileEdekInfo entry : batch) {
final INode inode = fsd.getInode(entry.getInodeId());
- Preconditions.checkNotNull(inode);
+ // no dir lock, so inode could be removed. no-op if so.
+ if (inode == null) {
+ NameNode.LOG.info("Cannot find inode {}, skip saving xattr for"
+ + " re-encryption", entry.getInodeId());
+ continue;
+ }
fsd.getEditLog().logSetXAttrs(inode.getFullPathName(),
inode.getXAttrFeature().getXAttrs(), false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/248d9b6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
index 729b894..01c2038 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
@@ -45,11 +45,11 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -112,8 +112,7 @@ public class ReencryptionHandler implements Runnable {
private ExecutorCompletionService<ReencryptionTask> batchService;
private BlockingQueue<Runnable> taskQueue;
// protected by ReencryptionHandler object lock
- private final Map<Long, ZoneSubmissionTracker> submissions =
- new ConcurrentHashMap<>();
+ private final Map<Long, ZoneSubmissionTracker> submissions = new HashMap<>();
// The current batch that the handler is working on. Handler is designed to
// be single-threaded, see class javadoc for more details.
@@ -132,8 +131,10 @@ public class ReencryptionHandler implements Runnable {
*/
void stopThreads() {
assert dir.hasWriteLock();
- for (ZoneSubmissionTracker zst : submissions.values()) {
- zst.cancelAllTasks();
+ synchronized (this) {
+ for (ZoneSubmissionTracker zst : submissions.values()) {
+ zst.cancelAllTasks();
+ }
}
if (updaterExecutor != null) {
updaterExecutor.shutdownNow();
@@ -269,33 +270,34 @@ public class ReencryptionHandler implements Runnable {
throw new IOException("Zone " + zoneName + " is not under re-encryption");
}
zs.cancel();
- ZoneSubmissionTracker zst = submissions.get(zoneId);
- if (zst != null) {
- zst.cancelAllTasks();
- }
+ removeZoneTrackerStopTasks(zoneId);
}
void removeZone(final long zoneId) {
assert dir.hasWriteLock();
LOG.info("Removing zone {} from re-encryption.", zoneId);
- ZoneSubmissionTracker zst = submissions.get(zoneId);
+ removeZoneTrackerStopTasks(zoneId);
+ getReencryptionStatus().removeZone(zoneId);
+ }
+
+ synchronized private void removeZoneTrackerStopTasks(final long zoneId) {
+ final ZoneSubmissionTracker zst = submissions.get(zoneId);
if (zst != null) {
zst.cancelAllTasks();
+ submissions.remove(zoneId);
}
- submissions.remove(zoneId);
- getReencryptionStatus().removeZone(zoneId);
}
ZoneSubmissionTracker getTracker(final long zoneId) {
- dir.hasReadLock();
+ assert dir.hasReadLock();
return unprotectedGetTracker(zoneId);
}
/**
- * get the tracker without holding the FSDirectory lock. This is only used for
- * testing, when updater checks about pausing.
+ * Get the tracker without holding the FSDirectory lock.
+ * The submissions object is protected by object lock.
*/
- ZoneSubmissionTracker unprotectedGetTracker(final long zoneId) {
+ synchronized ZoneSubmissionTracker unprotectedGetTracker(final long zoneId) {
return submissions.get(zoneId);
}
@@ -308,16 +310,19 @@ public class ReencryptionHandler implements Runnable {
*
* @param zoneId
*/
- void addDummyTracker(final long zoneId) {
+ void addDummyTracker(final long zoneId, ZoneSubmissionTracker zst) {
assert dir.hasReadLock();
- assert !submissions.containsKey(zoneId);
- final ZoneSubmissionTracker zst = new ZoneSubmissionTracker();
+ if (zst == null) {
+ zst = new ZoneSubmissionTracker();
+ }
zst.setSubmissionDone();
- Future future = batchService.submit(
+ final Future future = batchService.submit(
new EDEKReencryptCallable(zoneId, new ReencryptionBatch(), this));
zst.addTask(future);
- submissions.put(zoneId, zst);
+ synchronized (this) {
+ submissions.put(zoneId, zst);
+ }
}
/**
@@ -351,6 +356,8 @@ public class ReencryptionHandler implements Runnable {
}
LOG.info("Executing re-encrypt commands on zone {}. Current zones:{}",
zoneId, getReencryptionStatus());
+ getReencryptionStatus().markZoneStarted(zoneId);
+ resetSubmissionTracker(zoneId);
} finally {
dir.readUnlock();
}
@@ -392,7 +399,6 @@ public class ReencryptionHandler implements Runnable {
readLock();
try {
- getReencryptionStatus().markZoneStarted(zoneId);
zoneNode = dir.getInode(zoneId);
// start re-encrypting the zone from the beginning
if (zoneNode == null) {
@@ -428,6 +434,20 @@ public class ReencryptionHandler implements Runnable {
}
}
+ /**
+ * Reset the zone submission tracker for re-encryption.
+ * @param zoneId
+ */
+ synchronized private void resetSubmissionTracker(final long zoneId) {
+ ZoneSubmissionTracker zst = submissions.get(zoneId);
+ if (zst == null) {
+ zst = new ZoneSubmissionTracker();
+ submissions.put(zoneId, zst);
+ } else {
+ zst.reset();
+ }
+ }
+
List<XAttr> completeReencryption(final INode zoneNode) throws IOException {
assert dir.hasWriteLock();
assert dir.getFSNamesystem().hasWriteLock();
@@ -437,8 +457,9 @@ public class ReencryptionHandler implements Runnable {
LOG.info("Re-encryption completed on zone {}. Re-encrypted {} files,"
+ " failures encountered: {}.", zoneNode.getFullPathName(),
zs.getFilesReencrypted(), zs.getNumReencryptionFailures());
- // This also removes the zone from reencryptionStatus
- submissions.remove(zoneId);
+ synchronized (this) {
+ submissions.remove(zoneId);
+ }
return FSDirEncryptionZoneOp
.updateReencryptionFinish(dir, INodesInPath.fromINode(zoneNode), zs);
}
@@ -562,10 +583,13 @@ public class ReencryptionHandler implements Runnable {
if (currentBatch.isEmpty()) {
return;
}
- ZoneSubmissionTracker zst = submissions.get(zoneId);
- if (zst == null) {
- zst = new ZoneSubmissionTracker();
- submissions.put(zoneId, zst);
+ ZoneSubmissionTracker zst;
+ synchronized (this) {
+ zst = submissions.get(zoneId);
+ if (zst == null) {
+ zst = new ZoneSubmissionTracker();
+ submissions.put(zoneId, zst);
+ }
}
Future future = batchService
.submit(new EDEKReencryptCallable(zoneId, currentBatch, this));
@@ -821,19 +845,13 @@ public class ReencryptionHandler implements Runnable {
// 2. if tasks are piling up on the updater, don't create new callables
// until the queue size goes down.
final int maxTasksPiled = Runtime.getRuntime().availableProcessors() * 2;
- int totalTasks = 0;
- for (ZoneSubmissionTracker zst : submissions.values()) {
- totalTasks += zst.getTasks().size();
- }
- if (totalTasks >= maxTasksPiled) {
+ int numTasks = numTasksSubmitted();
+ if (numTasks >= maxTasksPiled) {
LOG.debug("Re-encryption handler throttling because total tasks pending"
- + " re-encryption updater is {}", totalTasks);
- while (totalTasks >= maxTasksPiled) {
+ + " re-encryption updater is {}", numTasks);
+ while (numTasks >= maxTasksPiled) {
Thread.sleep(500);
- totalTasks = 0;
- for (ZoneSubmissionTracker zst : submissions.values()) {
- totalTasks += zst.getTasks().size();
- }
+ numTasks = numTasksSubmitted();
}
}
@@ -864,6 +882,14 @@ public class ReencryptionHandler implements Runnable {
throttleTimerLocked.reset();
}
+ private synchronized int numTasksSubmitted() {
+ int ret = 0;
+ for (ZoneSubmissionTracker zst : submissions.values()) {
+ ret += zst.getTasks().size();
+ }
+ return ret;
+ }
+
/**
* Process an Inode for re-encryption. Add to current batch if it's a file,
* no-op otherwise.
@@ -877,7 +903,7 @@ public class ReencryptionHandler implements Runnable {
*/
private boolean reencryptINode(final INode inode, final String ezKeyVerName)
throws IOException, InterruptedException {
- dir.hasReadLock();
+ assert dir.hasReadLock();
if (LOG.isTraceEnabled()) {
LOG.trace("Processing {} for re-encryption", inode.getFullPathName());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/248d9b6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
index d641ea1..3b7badb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
@@ -93,6 +93,13 @@ public final class ReencryptionUpdater implements Runnable {
numFutureDone = 0;
}
+ void reset() {
+ submissionDone = false;
+ tasks.clear();
+ numCheckpointed = 0;
+ numFutureDone = 0;
+ }
+
LinkedList<Future> getTasks() {
return tasks;
}
@@ -238,12 +245,12 @@ public final class ReencryptionUpdater implements Runnable {
void markZoneSubmissionDone(final long zoneId)
throws IOException, InterruptedException {
final ZoneSubmissionTracker tracker = handler.getTracker(zoneId);
- if (tracker != null) {
+ if (tracker != null && !tracker.getTasks().isEmpty()) {
tracker.submissionDone = true;
} else {
// Caller thinks submission is done, but no tasks submitted - meaning
// no files in the EZ need to be re-encrypted. Complete directly.
- handler.addDummyTracker(zoneId);
+ handler.addDummyTracker(zoneId, tracker);
}
}
@@ -289,6 +296,7 @@ public final class ReencryptionUpdater implements Runnable {
LOG.debug(
"Updating file xattrs for re-encrypting zone {}," + " starting at {}",
zoneNodePath, task.batch.getFirstFilePath());
+ final int batchSize = task.batch.size();
for (Iterator<FileEdekInfo> it = task.batch.getBatch().iterator();
it.hasNext();) {
FileEdekInfo entry = it.next();
@@ -342,7 +350,7 @@ public final class ReencryptionUpdater implements Runnable {
}
LOG.info("Updated xattrs on {}({}) files in zone {} for re-encryption,"
- + " starting:{}.", task.numFilesUpdated, task.batch.size(),
+ + " starting:{}.", task.numFilesUpdated, batchSize,
zoneNodePath, task.batch.getFirstFilePath());
}
task.processed = true;
@@ -377,6 +385,9 @@ public final class ReencryptionUpdater implements Runnable {
ListIterator<Future> iter = tasks.listIterator();
while (iter.hasNext()) {
Future<ReencryptionTask> curr = iter.next();
+ if (curr.isCancelled()) {
+ break;
+ }
if (!curr.isDone() || !curr.get().processed) {
// still has earlier tasks not completed, skip here.
break;
@@ -411,12 +422,12 @@ public final class ReencryptionUpdater implements Runnable {
final Future<ReencryptionTask> completed = batchService.take();
throttle();
checkPauseForTesting();
- ReencryptionTask task = completed.get();
if (completed.isCancelled()) {
- LOG.debug("Skipped canceled re-encryption task for zone {}, last: {}",
- task.zoneId, task.lastFile);
+ // Ignore canceled zones. The cancellation is edit-logged by the handler.
+ LOG.debug("Skipped a canceled re-encryption task");
return;
}
+ final ReencryptionTask task = completed.get();
boolean shouldRetry;
do {
@@ -465,7 +476,11 @@ public final class ReencryptionUpdater implements Runnable {
task.batch.size(), task.batch.getFirstFilePath());
final ZoneSubmissionTracker tracker =
handler.getTracker(zoneNode.getId());
- Preconditions.checkNotNull(tracker, "zone tracker not found " + zonePath);
+ if (tracker == null) {
+ // re-encryption canceled.
+ LOG.info("Re-encryption was canceled.");
+ return;
+ }
tracker.numFutureDone++;
EncryptionFaultInjector.getInstance().reencryptUpdaterProcessOneTask();
processTaskEntries(zonePath, task);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/248d9b6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
index 4bf6aa4..aca9a73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
@@ -1467,8 +1467,8 @@ public class TestReencryption {
assertEquals(5, getZoneStatus(zone.toString()).getFilesReencrypted());
}
- @Test
- public void testCancelFuture() throws Exception {
+ private void cancelFutureDuringReencryption(final Path zone)
+ throws Exception {
final AtomicBoolean callableRunning = new AtomicBoolean(false);
class MyInjector extends EncryptionFaultInjector {
private volatile int exceptionCount = 0;
@@ -1498,8 +1498,6 @@ public class TestReencryption {
* /dir/f
*/
final int len = 8196;
- final Path zoneParent = new Path("/zones");
- final Path zone = new Path(zoneParent, "zone");
fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
for (int i = 0; i < 10; ++i) {
@@ -1549,6 +1547,59 @@ public class TestReencryption {
}
@Test
+ public void testCancelFutureThenReencrypt() throws Exception {
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ cancelFutureDuringReencryption(zone);
+
+ // make sure new re-encryption after cancellation works.
+ getEzManager().resumeReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForZoneCompletes(zone.toString());
+ final RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ final ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertFalse(zs.isCanceled());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(10, zs.getFilesReencrypted());
+ }
+
+ @Test
+ public void testCancelFutureThenRestart() throws Exception {
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ cancelFutureDuringReencryption(zone);
+
+ // restart, and check status.
+ restartClusterDisableReencrypt();
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.isCanceled());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(0, zs.getFilesReencrypted());
+
+ // verify re-encryption works after restart.
+ getEzManager().resumeReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForZoneCompletes(zone.toString());
+ it = dfsAdmin.listReencryptionStatus();
+ zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertFalse(zs.isCanceled());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(10, zs.getFilesReencrypted());
+ }
+
+ @Test
public void testReencryptCancelForUpdater() throws Exception {
/* Setup test dir:
* /zones/zone/[0-9]
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] hadoop git commit: YARN-7359.
TestAppManager.testQueueSubmitWithNoPermission() should be scheduler
agnostic. (Contributed by Haibo Chen)
Posted by ae...@apache.org.
YARN-7359. TestAppManager.testQueueSubmitWithNoPermission() should be scheduler agnostic. (Contributed by Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b4b0187
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b4b0187
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b4b0187
Branch: refs/heads/HDFS-7240
Commit: 7b4b0187806601e33f5a88d48991e7c12ee4419f
Parents: ca8ddc6
Author: Yufei Gu <yu...@apache.org>
Authored: Thu Oct 19 16:51:29 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Thu Oct 19 16:51:47 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/server/resourcemanager/TestAppManager.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b4b0187/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index b24a309..8179321 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -305,8 +305,6 @@ public class TestAppManager{
@Test
public void testQueueSubmitWithNoPermission() throws IOException {
YarnConfiguration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.RM_SCHEDULER,
- CapacityScheduler.class.getCanonicalName());
conf.set(PREFIX + "root.acl_submit_applications", " ");
conf.set(PREFIX + "root.acl_administer_queue", " ");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] hadoop git commit: HDFS-12603. Enable async edit logging by
default. Contributed by Andrew Wang.
Posted by ae...@apache.org.
HDFS-12603. Enable async edit logging by default. Contributed by Andrew Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/035c6ee5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/035c6ee5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/035c6ee5
Branch: refs/heads/HDFS-7240
Commit: 035c6ee587e444550af6420676e4cee049e09869
Parents: 9fcc3a1
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Oct 16 09:43:39 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Oct 16 09:51:10 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 2 +-
.../src/main/resources/hdfs-default.xml | 2 +-
.../namenode/ha/TestFailureToReadEdits.java | 25 ++++++++++++++------
4 files changed, 21 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 1f96763..395b192 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging";
- public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
+ public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index af2a5af9..7ca63f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
*
* In a non-HA setup:
*
- * The log starts in UNITIALIZED state upon construction. Once it's
+ * The log starts in UNINITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3491ed2..3752578 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4105,7 +4105,7 @@
<property>
<name>dfs.namenode.edits.asynclogging</name>
- <value>false</value>
+ <value>true</value>
<description>
If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/035c6ee5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index 93c717c..a37631f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random();
private final TestType clusterType;
+ private final boolean useAsyncEditLogging;
private Configuration conf;
private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@@ -88,18 +89,26 @@ public class TestFailureToReadEdits {
};
/**
- * Run this suite of tests both for QJM-based HA and for file-based
- * HA.
+ * Run this suite of tests for {QJM-based, file-based HA} x {async
+ * edit logging enabled, disabled}.
+ *
+ * TODO: Enable the test cases with async edit logging on. See HDFS-12603
+ * and HDFS-12660.
*/
@Parameters
public static Iterable<Object[]> data() {
- return Arrays.asList(new Object[][] {
- { TestType.SHARED_DIR_HA },
- { TestType.QJM_HA } });
+ return Arrays.asList(new Object[][]{
+ {TestType.SHARED_DIR_HA, Boolean.FALSE},
+ //{TestType.SHARED_DIR_HA, Boolean.TRUE},
+ {TestType.QJM_HA, Boolean.FALSE},
+ //{TestType.QJM_HA, Boolean.TRUE},
+ });
}
-
- public TestFailureToReadEdits(TestType clusterType) {
+
+ public TestFailureToReadEdits(TestType clusterType, Boolean
+ useAsyncEditLogging) {
this.clusterType = clusterType;
+ this.useAsyncEditLogging = useAsyncEditLogging;
}
@Before
@@ -109,6 +118,8 @@ public class TestFailureToReadEdits {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
+ useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] hadoop git commit: HDFS-12249. dfsadmin -metaSave to output
maintenance mode blocks. Contributed by Wellington Chevreuil.
Posted by ae...@apache.org.
HDFS-12249. dfsadmin -metaSave to output maintenance mode blocks. Contributed by Wellington Chevreuil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f2a6912
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f2a6912
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f2a6912
Branch: refs/heads/HDFS-7240
Commit: 0f2a69127aa40b7dccdf74ff63e2205a10ae8998
Parents: 9e77dc2
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Oct 23 15:23:52 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Oct 23 15:24:34 2017 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 12 +-
.../blockmanagement/TestBlockManager.java | 111 +++++++++++++++++++
2 files changed, 119 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f2a6912/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8b0f094..6cd67f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -833,10 +833,12 @@ public class BlockManager implements BlockStatsMXBean {
// l: == live:, d: == decommissioned c: == corrupt e: == excess
out.print(block + ((usableReplicas > 0)? "" : " MISSING") +
" (replicas:" +
- " l: " + numReplicas.liveReplicas() +
- " d: " + numReplicas.decommissionedAndDecommissioning() +
- " c: " + numReplicas.corruptReplicas() +
- " e: " + numReplicas.excessReplicas() + ") ");
+ " live: " + numReplicas.liveReplicas() +
+ " decommissioning and decommissioned: " +
+ numReplicas.decommissionedAndDecommissioning() +
+ " corrupt: " + numReplicas.corruptReplicas() +
+ " in excess: " + numReplicas.excessReplicas() +
+ " maintenance mode: " + numReplicas.maintenanceReplicas() + ") ");
Collection<DatanodeDescriptor> corruptNodes =
corruptReplicas.getNodes(block);
@@ -849,6 +851,8 @@ public class BlockManager implements BlockStatsMXBean {
} else if (node.isDecommissioned() ||
node.isDecommissionInProgress()) {
state = "(decommissioned)";
+ } else if (node.isMaintenance() || node.isInMaintenance()){
+ state = "(maintenance)";
}
if (storage.areBlockContentsStale()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f2a6912/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 10289ed..4d01d80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1511,4 +1511,115 @@ public class TestBlockManager {
return blockInfo;
}
+ private BlockInfo makeBlockReplicasMaintenance(long blockId,
+ List<DatanodeDescriptor> nodesList) throws IOException {
+ long inodeId = ++mockINodeId;
+ final INodeFile bc = TestINodeFile.createINodeFile(inodeId);
+
+ BlockInfo blockInfo = blockOnNodes(blockId, nodesList);
+ blockInfo.setReplication((short) 3);
+ blockInfo.setBlockCollectionId(inodeId);
+
+ Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId);
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ nodesList.get(0).setInMaintenance();
+ BlockCollection mockedBc = Mockito.mock(BlockCollection.class);
+ Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo});
+ bm.checkRedundancy(mockedBc);
+ return blockInfo;
+ }
+
+ @Test
+ public void testMetaSaveInMaintenanceReplicas() throws Exception {
+ List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
+ List<DatanodeDescriptor> origNodes = getNodes(origStorages);
+ BlockInfo block = makeBlockReplicasMaintenance(0, origNodes);
+ File file = new File("test.log");
+ PrintWriter out = new PrintWriter(file);
+ bm.metaSave(out);
+ out.flush();
+ FileInputStream fstream = new FileInputStream(file);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ StringBuffer buffer = new StringBuffer();
+ String line;
+ try {
+ while ((line = reader.readLine()) != null) {
+ buffer.append(line);
+ System.out.println(line);
+ }
+ String output = buffer.toString();
+ assertTrue("Metasave output should not have reported " +
+ "missing blocks.",
+ output.contains("Metasave: Blocks currently missing: 0"));
+ assertTrue("There should be 1 block waiting for reconstruction",
+ output.contains("Metasave: Blocks waiting for reconstruction: 1"));
+ String blockNameGS = block.getBlockName() + "_" +
+ block.getGenerationStamp();
+ assertTrue("Block " + blockNameGS +
+ " should be list as maintenance.",
+ output.contains(blockNameGS + " (replicas: live: 1 decommissioning " +
+ "and decommissioned: 0 corrupt: 0 in excess: " +
+ "0 maintenance mode: 1)"));
+ } finally {
+ reader.close();
+ file.delete();
+ }
+ }
+
+ private BlockInfo makeBlockReplicasDecommission(long blockId,
+ List<DatanodeDescriptor> nodesList) throws IOException {
+ long inodeId = ++mockINodeId;
+ final INodeFile bc = TestINodeFile.createINodeFile(inodeId);
+
+ BlockInfo blockInfo = blockOnNodes(blockId, nodesList);
+ blockInfo.setReplication((short) 3);
+ blockInfo.setBlockCollectionId(inodeId);
+
+ Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId);
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ nodesList.get(0).startDecommission();
+ BlockCollection mockedBc = Mockito.mock(BlockCollection.class);
+ Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo});
+ bm.checkRedundancy(mockedBc);
+ return blockInfo;
+ }
+
+ @Test
+ public void testMetaSaveDecommissioningReplicas() throws Exception {
+ List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
+ List<DatanodeDescriptor> origNodes = getNodes(origStorages);
+ BlockInfo block = makeBlockReplicasDecommission(0, origNodes);
+ File file = new File("test.log");
+ PrintWriter out = new PrintWriter(file);
+ bm.metaSave(out);
+ out.flush();
+ FileInputStream fstream = new FileInputStream(file);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ StringBuffer buffer = new StringBuffer();
+ String line;
+ try {
+ while ((line = reader.readLine()) != null) {
+ buffer.append(line);
+ }
+ String output = buffer.toString();
+ assertTrue("Metasave output should not have reported " +
+ "missing blocks.",
+ output.contains("Metasave: Blocks currently missing: 0"));
+ assertTrue("There should be 1 block waiting for reconstruction",
+ output.contains("Metasave: Blocks waiting for reconstruction: 1"));
+ String blockNameGS = block.getBlockName() + "_" +
+ block.getGenerationStamp();
+ assertTrue("Block " + blockNameGS +
+ " should be list as maintenance.",
+ output.contains(blockNameGS + " (replicas: live: 1 decommissioning " +
+ "and decommissioned: 1 corrupt: 0 in excess: " +
+ "0 maintenance mode: 0)"));
+ } finally {
+ reader.close();
+ file.delete();
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] hadoop git commit: YARN-7341.
TestRouterWebServiceUtil#testMergeMetrics is flakey. (Robert Kanter via Haibo
Chen)
Posted by ae...@apache.org.
YARN-7341. TestRouterWebServiceUtil#testMergeMetrics is flakey. (Robert Kanter via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acabc657
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acabc657
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acabc657
Branch: refs/heads/HDFS-7240
Commit: acabc657ff5433f36ce1b238cecd3a3b5bbe87ae
Parents: 31ebccc
Author: Haibo Chen <ha...@apache.org>
Authored: Tue Oct 17 10:15:53 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Oct 17 10:15:53 2017 -0700
----------------------------------------------------------------------
.../router/webapp/RouterWebServiceUtil.java | 36 +++++++++++---------
.../router/webapp/TestRouterWebServiceUtil.java | 17 ++++++---
2 files changed, 33 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acabc657/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index 5528e78..76435f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -406,22 +406,26 @@ public final class RouterWebServiceUtil {
metrics.setContainersPending(metrics.getPendingContainers()
+ metricsResponse.getPendingContainers());
- metrics.setTotalMB(metrics.getTotalMB() + metricsResponse.getTotalMB());
- metrics.setTotalVirtualCores(
- metrics.getTotalVirtualCores() + metrics.getTotalVirtualCores());
- metrics.setTotalNodes(metrics.getTotalNodes() + metrics.getTotalNodes());
- metrics.setLostNodes(metrics.getLostNodes() + metrics.getLostNodes());
- metrics.setUnhealthyNodes(
- metrics.getUnhealthyNodes() + metrics.getUnhealthyNodes());
- metrics.setDecommissioningNodes(
- metrics.getDecommissioningNodes() + metrics.getDecommissioningNodes());
- metrics.setDecommissionedNodes(
- metrics.getDecommissionedNodes() + metrics.getDecommissionedNodes());
- metrics.setRebootedNodes(
- metrics.getRebootedNodes() + metrics.getRebootedNodes());
- metrics.setActiveNodes(metrics.getActiveNodes() + metrics.getActiveNodes());
- metrics.setShutdownNodes(
- metrics.getShutdownNodes() + metrics.getShutdownNodes());
+ metrics.setTotalMB(metrics.getTotalMB()
+ + metricsResponse.getTotalMB());
+ metrics.setTotalVirtualCores(metrics.getTotalVirtualCores()
+ + metricsResponse.getTotalVirtualCores());
+ metrics.setTotalNodes(metrics.getTotalNodes()
+ + metricsResponse.getTotalNodes());
+ metrics.setLostNodes(metrics.getLostNodes()
+ + metricsResponse.getLostNodes());
+ metrics.setUnhealthyNodes(metrics.getUnhealthyNodes()
+ + metricsResponse.getUnhealthyNodes());
+ metrics.setDecommissioningNodes(metrics.getDecommissioningNodes()
+ + metricsResponse.getDecommissioningNodes());
+ metrics.setDecommissionedNodes(metrics.getDecommissionedNodes()
+ + metricsResponse.getDecommissionedNodes());
+ metrics.setRebootedNodes(metrics.getRebootedNodes()
+ + metricsResponse.getRebootedNodes());
+ metrics.setActiveNodes(metrics.getActiveNodes()
+ + metricsResponse.getActiveNodes());
+ metrics.setShutdownNodes(metrics.getShutdownNodes()
+ + metricsResponse.getShutdownNodes());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acabc657/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
index 7073b3b..edf3804 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
@@ -33,11 +33,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestI
import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Test class to validate RouterWebServiceUtil methods.
*/
public class TestRouterWebServiceUtil {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRouterWebServiceUtil.class);
private static final ApplicationId APPID1 = ApplicationId.newInstance(1, 1);
private static final ApplicationId APPID2 = ApplicationId.newInstance(2, 1);
@@ -407,8 +411,12 @@ public class TestRouterWebServiceUtil {
ClusterMetricsInfo metrics = new ClusterMetricsInfo();
ClusterMetricsInfo metricsResponse = new ClusterMetricsInfo();
- setUpClusterMetrics(metrics);
- setUpClusterMetrics(metricsResponse);
+ long seed = System.currentTimeMillis();
+ setUpClusterMetrics(metrics, seed);
+ // ensure that we don't reuse the same seed when setting up metricsResponse
+ // or it might mask bugs
+ seed += 1000000000;
+ setUpClusterMetrics(metricsResponse, seed);
ClusterMetricsInfo metricsClone = createClusterMetricsClone(metrics);
RouterWebServiceUtil.mergeMetrics(metrics, metricsResponse);
@@ -538,8 +546,9 @@ public class TestRouterWebServiceUtil {
}
- private void setUpClusterMetrics(ClusterMetricsInfo metrics) {
- Random rand = new Random(System.currentTimeMillis());
+ private void setUpClusterMetrics(ClusterMetricsInfo metrics, long seed) {
+ LOG.info("Using seed: " + seed);
+ Random rand = new Random(seed);
metrics.setAppsSubmitted(rand.nextInt(1000));
metrics.setAppsCompleted(rand.nextInt(1000));
metrics.setAppsPending(rand.nextInt(1000));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] hadoop git commit: HADOOP-14958. Fix source-level
compatibility after HADOOP-11252. Contributed by Junping Du.
Posted by ae...@apache.org.
HADOOP-14958. Fix source-level compatibility after HADOOP-11252. Contributed by Junping Du.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b016f08f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b016f08f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b016f08f
Branch: refs/heads/HDFS-7240
Commit: b016f08f67830ed3ca741bc6a10c3f5164781be5
Parents: 2523e1c
Author: Junping Du <ju...@apache.org>
Authored: Wed Oct 18 15:06:30 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Wed Oct 18 15:06:30 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ipc/Client.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b016f08f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index c9ac615..a0417d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -212,7 +212,8 @@ public class Client implements AutoCloseable {
* @param conf Configuration
* @param pingInterval the ping interval
*/
- static final void setPingInterval(Configuration conf, int pingInterval) {
+ public static final void setPingInterval(Configuration conf,
+ int pingInterval) {
conf.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, pingInterval);
}
@@ -223,7 +224,7 @@ public class Client implements AutoCloseable {
* @param conf Configuration
* @return the ping interval
*/
- static final int getPingInterval(Configuration conf) {
+ public static final int getPingInterval(Configuration conf) {
return conf.getInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,
CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] hadoop git commit: HDFS-12502. nntop should support a
category based on FilesInGetListingOps.
Posted by ae...@apache.org.
HDFS-12502. nntop should support a category based on FilesInGetListingOps.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60bfee27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60bfee27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60bfee27
Branch: refs/heads/HDFS-7240
Commit: 60bfee270ed3a653c44c0bc92396167b5022df6e
Parents: b016f08
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Oct 18 23:51:24 2017 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Oct 18 23:51:24 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystem.java | 5 ++++
.../server/namenode/top/metrics/TopMetrics.java | 30 +++++++++++++++-----
.../server/namenode/metrics/TestTopMetrics.java | 11 +++++--
3 files changed, 36 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bfee27/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e8d7161..1e8f319 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3673,6 +3673,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
readUnlock(operationName);
}
logAuditEvent(true, operationName, src);
+ if (topConf.isEnabled && isAuditEnabled() && isExternalInvocation()
+ && dl != null && Server.getRemoteUser() != null) {
+ topMetrics.reportFilesInGetListing(Server.getRemoteUser().toString(),
+ dl.getPartialListing().length);
+ }
return dl;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bfee27/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
index 2719c88..3d8dd19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
@@ -70,6 +70,14 @@ public class TopMetrics implements MetricsSource {
public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class);
public static final String TOPMETRICS_METRICS_SOURCE_NAME =
"NNTopUserOpCounts";
+ /**
+ * In addition to counts of different RPC calls, NNTop also reports top
+ * users listing large directories (measured by the number of files involved
+ * in listing operations from the user). This is important because the CPU
+ * and GC overhead of a listing operation grows linearly with the number of
+ * files involved. This category in NNTop is {@link #FILES_IN_GETLISTING}.
+ */
+ public static final String FILES_IN_GETLISTING = "filesInGetListing";
private final boolean isMetricsSourceEnabled;
private static void logConf(Configuration conf) {
@@ -123,22 +131,30 @@ public class TopMetrics implements MetricsSource {
public void report(boolean succeeded, String userName, InetAddress addr,
String cmd, String src, String dst, FileStatus status) {
// currently nntop only makes use of the username and the command
- report(userName, cmd);
+ report(userName, cmd, 1);
}
- public void report(String userName, String cmd) {
+ public void reportFilesInGetListing(String userName, int numFiles) {
+ report(userName, FILES_IN_GETLISTING, numFiles);
+ }
+
+ public void report(String userName, String cmd, int delta) {
long currTime = Time.monotonicNow();
- report(currTime, userName, cmd);
+ report(currTime, userName, cmd, delta);
}
- public void report(long currTime, String userName, String cmd) {
+ public void report(long currTime, String userName, String cmd, int delta) {
LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName);
userName = UserGroupInformation.trimLoginMethod(userName);
for (RollingWindowManager rollingWindowManager : rollingWindowManagers
.values()) {
- rollingWindowManager.recordMetric(currTime, cmd, userName, 1);
- rollingWindowManager.recordMetric(currTime,
- TopConf.ALL_CMDS, userName, 1);
+ rollingWindowManager.recordMetric(currTime, cmd, userName, delta);
+ // Increase the number of all RPC calls by the user, unless the report
+ // is for the number of files in a listing operation.
+ if (!cmd.equals(FILES_IN_GETLISTING)) {
+ rollingWindowManager.recordMetric(currTime,
+ TopConf.ALL_CMDS, userName, delta);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bfee27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
index 4d3a4f0..cda4b14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.metrics2.lib.Interns;
import org.junit.Test;
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
+import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.FILES_IN_GETLISTING;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -41,9 +42,10 @@ public class TestTopMetrics {
TopMetrics topMetrics = new TopMetrics(conf,
topConf.nntopReportingPeriodsMs);
// Dummy command
- topMetrics.report("test", "listStatus");
- topMetrics.report("test", "listStatus");
- topMetrics.report("test", "listStatus");
+ topMetrics.report("test", "listStatus", 1);
+ topMetrics.report("test", "listStatus", 1);
+ topMetrics.report("test", "listStatus", 1);
+ topMetrics.report("test", FILES_IN_GETLISTING, 1000);
MetricsRecordBuilder rb = getMetrics(topMetrics);
MetricsCollector mc = rb.parent();
@@ -59,5 +61,8 @@ public class TestTopMetrics {
verify(rb, times(3)).addCounter(Interns.info("op=listStatus." +
"user=test.count", "Total operations performed by user"), 3L);
+
+ verify(rb, times(3)).addCounter(Interns.info("op=" + FILES_IN_GETLISTING +
+ ".user=test.count", "Total operations performed by user"), 1000L);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] hadoop git commit: HDFS-12614.
FSPermissionChecker#getINodeAttrs() throws NPE when INodeAttributesProvider
configured.
Posted by ae...@apache.org.
HDFS-12614. FSPermissionChecker#getINodeAttrs() throws NPE when INodeAttributesProvider configured.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b406d8e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b406d8e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b406d8e3
Branch: refs/heads/HDFS-7240
Commit: b406d8e3755d24ce72c443fd893a5672fd56babc
Parents: e906108
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Mon Oct 16 17:42:41 2017 -0700
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Mon Oct 16 17:42:41 2017 -0700
----------------------------------------------------------------------
.../server/namenode/FSPermissionChecker.java | 12 +++-
.../namenode/TestINodeAttributeProvider.java | 60 ++++++++++++++------
2 files changed, 54 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b406d8e3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index f745a6c..c854b49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -275,8 +275,16 @@ class FSPermissionChecker implements AccessControlEnforcer {
INodeAttributes inodeAttrs = inode.getSnapshotINode(snapshotId);
if (getAttributesProvider() != null) {
String[] elements = new String[pathIdx + 1];
- for (int i = 0; i < elements.length; i++) {
- elements[i] = DFSUtil.bytes2String(pathByNameArr[i]);
+ /**
+ * {@link INode#getPathComponents(String)} returns a null component
+ * for the root only path "/". Assign an empty string if so.
+ */
+ if (pathByNameArr.length == 1 && pathByNameArr[0] == null) {
+ elements[0] = "";
+ } else {
+ for (int i = 0; i < elements.length; i++) {
+ elements[i] = DFSUtil.bytes2String(pathByNameArr[i]);
+ }
}
inodeAttrs = getAttributesProvider().getAttributes(elements, inodeAttrs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b406d8e3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
index bbc5fa0..9c7dcd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
@@ -313,31 +313,59 @@ public class TestINodeAttributeProvider {
testBypassProviderHelper(users, HDFS_PERMISSION, true);
}
- @Test
- public void testCustomProvider() throws Exception {
+ private void verifyFileStatus(UserGroupInformation ugi) throws IOException {
FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
- fs.mkdirs(new Path("/user/xxx"));
- FileStatus status = fs.getFileStatus(new Path("/user/xxx"));
- Assert.assertEquals(System.getProperty("user.name"), status.getOwner());
+
+ FileStatus status = fs.getFileStatus(new Path("/"));
+ LOG.info("Path '/' is owned by: "
+ + status.getOwner() + ":" + status.getGroup());
+
+ Path userDir = new Path("/user/" + ugi.getShortUserName());
+ fs.mkdirs(userDir);
+ status = fs.getFileStatus(userDir);
+ Assert.assertEquals(ugi.getShortUserName(), status.getOwner());
Assert.assertEquals("supergroup", status.getGroup());
Assert.assertEquals(new FsPermission((short) 0755), status.getPermission());
- fs.mkdirs(new Path("/user/authz"));
- Path p = new Path("/user/authz");
- status = fs.getFileStatus(p);
+
+ Path authzDir = new Path("/user/authz");
+ fs.mkdirs(authzDir);
+ status = fs.getFileStatus(authzDir);
Assert.assertEquals("foo", status.getOwner());
Assert.assertEquals("bar", status.getGroup());
Assert.assertEquals(new FsPermission((short) 0770), status.getPermission());
- AclStatus aclStatus = fs.getAclStatus(p);
+
+ AclStatus aclStatus = fs.getAclStatus(authzDir);
Assert.assertEquals(1, aclStatus.getEntries().size());
- Assert.assertEquals(AclEntryType.GROUP, aclStatus.getEntries().get(0)
- .getType());
- Assert.assertEquals("xxx", aclStatus.getEntries().get(0)
- .getName());
- Assert.assertEquals(FsAction.ALL, aclStatus.getEntries().get(0)
- .getPermission());
- Map<String, byte[]> xAttrs = fs.getXAttrs(p);
+ Assert.assertEquals(AclEntryType.GROUP,
+ aclStatus.getEntries().get(0).getType());
+ Assert.assertEquals("xxx",
+ aclStatus.getEntries().get(0).getName());
+ Assert.assertEquals(FsAction.ALL,
+ aclStatus.getEntries().get(0).getPermission());
+ Map<String, byte[]> xAttrs = fs.getXAttrs(authzDir);
Assert.assertTrue(xAttrs.containsKey("user.test"));
Assert.assertEquals(2, xAttrs.get("user.test").length);
}
+ /**
+ * With the custom provider configured, verify file status attributes.
+ * A superuser can bypass permission check while resolving paths. So,
+ * verify file status for both superuser and non-superuser.
+ */
+ @Test
+ public void testCustomProvider() throws Exception {
+ final UserGroupInformation[] users = new UserGroupInformation[]{
+ UserGroupInformation.createUserForTesting(
+ System.getProperty("user.name"), new String[]{"supergroup"}),
+ UserGroupInformation.createUserForTesting(
+ "normaluser", new String[]{"normalusergroup"}),
+ };
+
+ for (final UserGroupInformation user : users) {
+ user.doAs((PrivilegedExceptionAction<Object>) () -> {
+ verifyFileStatus(user);
+ return null;
+ });
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] hadoop git commit: HDFS-12485. expunge may fail to remove
trash from encryption zone. Contributed by Wei-Chiu Chuang.
Posted by ae...@apache.org.
HDFS-12485. expunge may fail to remove trash from encryption zone. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8dbc8909
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8dbc8909
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8dbc8909
Branch: refs/heads/HDFS-7240
Commit: 8dbc8909c92d502d10a7f94d1de3171878a43b04
Parents: 21bc855
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Oct 16 12:57:48 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Oct 16 12:57:48 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java | 3 +--
.../apache/hadoop/hdfs/TestTrashWithSecureEncryptionZones.java | 1 -
2 files changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dbc8909/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index c9f4490..a8a5cfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2783,8 +2783,7 @@ public class DistributedFileSystem extends FileSystem {
}
}
} else {
- Path userTrash = new Path(ezTrashRoot, System.getProperty(
- "user.name"));
+ Path userTrash = new Path(ezTrashRoot, dfs.ugi.getShortUserName());
try {
ret.add(getFileStatus(userTrash));
} catch (FileNotFoundException ignored) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dbc8909/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestTrashWithSecureEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestTrashWithSecureEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestTrashWithSecureEncryptionZones.java
index 314adfb..a8e2a71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestTrashWithSecureEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestTrashWithSecureEncryptionZones.java
@@ -225,7 +225,6 @@ public class TestTrashWithSecureEncryptionZones {
clientConf = new Configuration(conf);
clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
shell = new FsShell(clientConf);
- System.setProperty("user.name", HDFS_USER_NAME);
}
@AfterClass
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] hadoop git commit: HADOOP-14880. [KMS] Document&test missing
KMS client side configs. Contributed by Gabor Bota.
Posted by ae...@apache.org.
HADOOP-14880. [KMS] Document&test missing KMS client side configs. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97c70c7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97c70c7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97c70c7a
Branch: refs/heads/HDFS-7240
Commit: 97c70c7ac6881f87eee1575bcbdd28b31ecac231
Parents: 60bfee2
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Oct 19 06:02:13 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Oct 19 06:02:13 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/crypto/key/kms/KMSClientProvider.java | 8 +++-----
.../org/apache/hadoop/fs/CommonConfigurationKeysPublic.java | 9 +++++++++
.../hadoop-common/src/main/resources/core-default.xml | 8 ++++++++
.../org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 3 ++-
4 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c70c7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index c514beb..c324cd7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -121,10 +121,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private static final String CONFIG_PREFIX = "hadoop.security.kms.client.";
- /* It's possible to specify a timeout, in seconds, in the config file */
- public static final String TIMEOUT_ATTR = CONFIG_PREFIX + "timeout";
- public static final int DEFAULT_TIMEOUT = 60;
-
/* Number of times to retry authentication in the event of auth failure
* (normally happens due to stale authToken)
*/
@@ -361,7 +357,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
throw new IOException(ex);
}
}
- int timeout = conf.getInt(TIMEOUT_ATTR, DEFAULT_TIMEOUT);
+ int timeout = conf.getInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_SECONDS,
+ CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_DEFAULT);
authRetry = conf.getInt(AUTH_RETRY, DEFAULT_AUTH_RETRY);
configurator = new TimeoutConnConfigurator(timeout, sslFactory);
encKeyVersionQueue =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c70c7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 4fda2b8..3c8628c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -726,6 +726,15 @@ public class CommonConfigurationKeysPublic {
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
* core-default.xml</a>
*/
+ public static final String KMS_CLIENT_TIMEOUT_SECONDS =
+ "hadoop.security.kms.client.timeout";
+ public static final int KMS_CLIENT_TIMEOUT_DEFAULT = 60;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
/** Default value is the number of providers specified. */
public static final String KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY =
"hadoop.security.kms.client.failover.max.retries";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c70c7a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index bde7a85..8db9f44 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2340,6 +2340,14 @@
key will be dropped. Default = 12hrs
</description>
</property>
+<property>
+ <name>hadoop.security.kms.client.timeout</name>
+ <value>60</value>
+ <description>
+ Sets value for KMS client connection timeout, and the read timeout
+ to KMS servers.
+ </description>
+</property>
<property>
<name>hadoop.security.kms.client.failover.sleep.base.millis</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97c70c7a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 7605b28..f7ecf44 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.KMSDelegationToken;
import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
import org.apache.hadoop.crypto.key.kms.ValueQueue;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
@@ -1883,7 +1884,7 @@ public class TestKMS {
public void testKMSTimeout() throws Exception {
File confDir = getTestDir();
Configuration conf = createBaseKMSConf(confDir);
- conf.setInt(KMSClientProvider.TIMEOUT_ATTR, 1);
+ conf.setInt(CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_SECONDS, 1);
writeConf(confDir, conf);
ServerSocket sock;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] hadoop git commit: HDFS-12613. Native EC coder should
implement release() as idempotent function. (Lei (Eddy) Xu)
Posted by ae...@apache.org.
HDFS-12613. Native EC coder should implement release() as idempotent function. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31ebccc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31ebccc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31ebccc9
Branch: refs/heads/HDFS-7240
Commit: 31ebccc96238136560f4210bdf6766fe18e0650c
Parents: b406d8e
Author: Lei Xu <le...@apache.org>
Authored: Mon Oct 16 19:44:30 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Mon Oct 16 19:44:30 2017 -0700
----------------------------------------------------------------------
.../io/erasurecode/coder/ErasureCodingStep.java | 5 +-
.../erasurecode/coder/ErasureDecodingStep.java | 5 +-
.../erasurecode/coder/ErasureEncodingStep.java | 5 +-
.../coder/HHXORErasureDecodingStep.java | 12 ++--
.../coder/HHXORErasureEncodingStep.java | 10 +++-
.../io/erasurecode/coder/util/HHUtil.java | 4 +-
.../rawcoder/AbstractNativeRawDecoder.java | 14 ++++-
.../rawcoder/AbstractNativeRawEncoder.java | 14 ++++-
.../rawcoder/NativeRSRawDecoder.java | 11 ++--
.../rawcoder/NativeRSRawEncoder.java | 11 ++--
.../rawcoder/NativeXORRawDecoder.java | 14 +++--
.../rawcoder/NativeXORRawEncoder.java | 9 +--
.../rawcoder/RSLegacyRawDecoder.java | 6 +-
.../erasurecode/rawcoder/RawErasureDecoder.java | 17 ++++--
.../erasurecode/rawcoder/RawErasureEncoder.java | 16 +++--
.../apache/hadoop/io/erasurecode/jni_common.c | 5 +-
.../hadoop/io/erasurecode/jni_rs_decoder.c | 9 ++-
.../hadoop/io/erasurecode/jni_rs_encoder.c | 9 ++-
.../hadoop/io/erasurecode/jni_xor_decoder.c | 9 ++-
.../hadoop/io/erasurecode/jni_xor_encoder.c | 9 ++-
.../erasurecode/coder/TestErasureCoderBase.java | 18 +++++-
.../coder/TestHHErasureCoderBase.java | 10 +++-
.../rawcoder/RawErasureCoderBenchmark.java | 9 +--
.../erasurecode/rawcoder/TestDummyRawCoder.java | 15 ++++-
.../rawcoder/TestNativeRSRawCoder.java | 6 ++
.../rawcoder/TestNativeXORRawCoder.java | 7 +++
.../erasurecode/rawcoder/TestRawCoderBase.java | 61 ++++++++++++++++++--
.../hadoop/hdfs/DFSStripedOutputStream.java | 2 +-
.../hadoop/hdfs/PositionStripeReader.java | 3 +-
.../hadoop/hdfs/StatefulStripeReader.java | 3 +-
.../org/apache/hadoop/hdfs/StripeReader.java | 7 ++-
.../StripedBlockChecksumReconstructor.java | 2 +-
.../erasurecode/StripedBlockReconstructor.java | 2 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 6 +-
34 files changed, 269 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
index 9dd0aed..fb89d99 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
@@ -21,6 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECChunk;
+import java.io.IOException;
+
/**
* Erasure coding step that's involved in encoding/decoding of a block group.
*/
@@ -47,7 +49,8 @@ public interface ErasureCodingStep {
* @param inputChunks
* @param outputChunks
*/
- void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks);
+ void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+ throws IOException;
/**
* Notify erasure coder that all the chunks of input blocks are processed so
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
index ae396a2..24f5547 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import java.io.IOException;
+
/**
* Erasure decoding step, a wrapper of all the necessary information to perform
* a decoding step involved in the whole process of decoding a block group.
@@ -50,7 +52,8 @@ public class ErasureDecodingStep implements ErasureCodingStep {
}
@Override
- public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+ throws IOException {
rawDecoder.decode(inputChunks, erasedIndexes, outputChunks);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
index df4ed4b..5fc5c7a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import java.io.IOException;
+
/**
* Erasure encoding step, a wrapper of all the necessary information to perform
* an encoding step involved in the whole process of encoding a block group.
@@ -46,7 +48,8 @@ public class ErasureEncodingStep implements ErasureCodingStep {
}
@Override
- public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+ throws IOException {
rawEncoder.encode(inputChunks, outputChunks);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
index 98b8503..16a3c0f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.io.erasurecode.coder;
+import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -64,7 +65,8 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
}
@Override
- public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+ throws IOException {
if (erasedIndexes.length == 0) {
return;
}
@@ -74,7 +76,8 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
performCoding(inputBuffers, outputBuffers);
}
- private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs)
+ throws IOException {
final int numDataUnits = rsRawDecoder.getNumDataUnits();
final int numParityUnits = rsRawDecoder.getNumParityUnits();
final int numTotalUnits = numDataUnits + numParityUnits;
@@ -119,7 +122,7 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
private void doDecodeSingle(ByteBuffer[][] inputs, ByteBuffer[][] outputs,
int erasedLocationToFix, int bufSize,
- boolean isDirect) {
+ boolean isDirect) throws IOException {
final int numDataUnits = rsRawDecoder.getNumDataUnits();
final int numParityUnits = rsRawDecoder.getNumParityUnits();
final int subPacketSize = getSubPacketSize();
@@ -261,7 +264,8 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
private void doDecodeMultiAndParity(ByteBuffer[][] inputs,
ByteBuffer[][] outputs,
- int[] erasedLocationToFix, int bufSize) {
+ int[] erasedLocationToFix, int bufSize)
+ throws IOException {
final int numDataUnits = rsRawDecoder.getNumDataUnits();
final int numParityUnits = rsRawDecoder.getNumParityUnits();
final int numTotalUnits = numDataUnits + numParityUnits;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
index 11b1bf1..6a56442 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.io.erasurecode.coder;
+import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -56,13 +57,15 @@ public class HHXORErasureEncodingStep extends HHErasureCodingStep {
}
@Override
- public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
+ throws IOException {
ByteBuffer[] inputBuffers = ECChunk.toBuffers(inputChunks);
ByteBuffer[] outputBuffers = ECChunk.toBuffers(outputChunks);
performCoding(inputBuffers, outputBuffers);
}
- private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ private void performCoding(ByteBuffer[] inputs, ByteBuffer[] outputs)
+ throws IOException {
final int numDataUnits = this.rsRawEncoder.getNumDataUnits();
final int numParityUnits = this.rsRawEncoder.getNumParityUnits();
final int subSPacketSize = getSubPacketSize();
@@ -95,7 +98,8 @@ public class HHXORErasureEncodingStep extends HHErasureCodingStep {
doEncode(hhInputs, hhOutputs);
}
- private void doEncode(ByteBuffer[][] inputs, ByteBuffer[][] outputs) {
+ private void doEncode(ByteBuffer[][] inputs, ByteBuffer[][] outputs)
+ throws IOException {
final int numParityUnits = this.rsRawEncoder.getNumParityUnits();
// calc piggyBacks using first sub-packet
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
index cfb567e3..91d0241 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.io.erasurecode.coder.util;
+import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -64,7 +65,8 @@ public final class HHUtil {
int[] piggyBackIndex,
int numParityUnits,
int pgIndex,
- RawErasureEncoder encoder) {
+ RawErasureEncoder encoder)
+ throws IOException {
ByteBuffer[] emptyInput = new ByteBuffer[inputs.length];
ByteBuffer[] tempInput = new ByteBuffer[inputs.length];
int[] inputPositions = new int[inputs.length];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
index d6aa8bd..e845747 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.util.PerformanceAdvisory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -38,7 +39,12 @@ abstract class AbstractNativeRawDecoder extends RawErasureDecoder {
}
@Override
- protected void doDecode(ByteBufferDecodingState decodingState) {
+ protected synchronized void doDecode(ByteBufferDecodingState decodingState)
+ throws IOException {
+ if (nativeCoder == 0) {
+ throw new IOException(String.format("%s closed",
+ getClass().getSimpleName()));
+ }
int[] inputOffsets = new int[decodingState.inputs.length];
int[] outputOffsets = new int[decodingState.outputs.length];
@@ -63,10 +69,12 @@ abstract class AbstractNativeRawDecoder extends RawErasureDecoder {
protected abstract void performDecodeImpl(ByteBuffer[] inputs,
int[] inputOffsets, int dataLen,
int[] erased, ByteBuffer[] outputs,
- int[] outputOffsets);
+ int[] outputOffsets)
+ throws IOException;
@Override
- protected void doDecode(ByteArrayDecodingState decodingState) {
+ protected void doDecode(ByteArrayDecodingState decodingState)
+ throws IOException {
PerformanceAdvisory.LOG.debug("convertToByteBufferState is invoked, " +
"not efficiently. Please use direct ByteBuffer inputs/outputs");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
index 21805fe..cab5383 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.util.PerformanceAdvisory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -38,7 +39,12 @@ abstract class AbstractNativeRawEncoder extends RawErasureEncoder {
}
@Override
- protected void doEncode(ByteBufferEncodingState encodingState) {
+ protected synchronized void doEncode(ByteBufferEncodingState encodingState)
+ throws IOException {
+ if (nativeCoder == 0) {
+ throw new IOException(String.format("%s closed",
+ getClass().getSimpleName()));
+ }
int[] inputOffsets = new int[encodingState.inputs.length];
int[] outputOffsets = new int[encodingState.outputs.length];
int dataLen = encodingState.inputs[0].remaining();
@@ -60,10 +66,12 @@ abstract class AbstractNativeRawEncoder extends RawErasureEncoder {
protected abstract void performEncodeImpl(
ByteBuffer[] inputs, int[] inputOffsets,
- int dataLen, ByteBuffer[] outputs, int[] outputOffsets);
+ int dataLen, ByteBuffer[] outputs, int[] outputOffsets)
+ throws IOException;
@Override
- protected void doEncode(ByteArrayEncodingState encodingState) {
+ protected void doEncode(ByteArrayEncodingState encodingState)
+ throws IOException {
PerformanceAdvisory.LOG.debug("convertToByteBufferState is invoked, " +
"not efficiently. Please use direct ByteBuffer inputs/outputs");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
index ce4ec2b..8572222 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -39,14 +40,14 @@ public class NativeRSRawDecoder extends AbstractNativeRawDecoder {
}
@Override
- protected void performDecodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
- int dataLen, int[] erased,
- ByteBuffer[] outputs, int[] outputOffsets) {
+ protected synchronized void performDecodeImpl(
+ ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
+ ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets);
}
@Override
- public void release() {
+ public synchronized void release() {
destroyImpl();
}
@@ -59,7 +60,7 @@ public class NativeRSRawDecoder extends AbstractNativeRawDecoder {
private native void decodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
- ByteBuffer[] outputs, int[] outputOffsets);
+ ByteBuffer[] outputs, int[] outputOffsets) throws IOException;
private native void destroyImpl();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
index 70b5a46..754ec88 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -39,14 +40,14 @@ public class NativeRSRawEncoder extends AbstractNativeRawEncoder {
}
@Override
- protected void performEncodeImpl(
+ protected synchronized void performEncodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen,
- ByteBuffer[] outputs, int[] outputOffsets) {
+ ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets);
}
@Override
- public void release() {
+ public synchronized void release() {
destroyImpl();
}
@@ -58,8 +59,8 @@ public class NativeRSRawEncoder extends AbstractNativeRawEncoder {
private native void initImpl(int numDataUnits, int numParityUnits);
private native void encodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
- int dataLen, ByteBuffer[] outputs,
- int[] outputOffsets);
+ int dataLen, ByteBuffer[] outputs,
+ int[] outputOffsets) throws IOException;
private native void destroyImpl();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
index b6b1673..1763042 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -39,21 +40,26 @@ public class NativeXORRawDecoder extends AbstractNativeRawDecoder {
}
@Override
- protected void performDecodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
- int dataLen, int[] erased, ByteBuffer[] outputs, int[] outputOffsets) {
+ protected synchronized void performDecodeImpl(
+ ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
+ ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets);
}
@Override
- public void release() {
+ public synchronized void release() {
destroyImpl();
}
private native void initImpl(int numDataUnits, int numParityUnits);
+ /**
+ * Native implementation of decoding.
+ * @throws IOException if the decoder is closed.
+ */
private native void decodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased,
- ByteBuffer[] outputs, int[] outputOffsets);
+ ByteBuffer[] outputs, int[] outputOffsets) throws IOException;
private native void destroyImpl();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
index 9b4b449..7f4265b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -39,14 +40,14 @@ public class NativeXORRawEncoder extends AbstractNativeRawEncoder {
}
@Override
- protected void performEncodeImpl(
+ protected synchronized void performEncodeImpl(
ByteBuffer[] inputs, int[] inputOffsets, int dataLen,
- ByteBuffer[] outputs, int[] outputOffsets) {
+ ByteBuffer[] outputs, int[] outputOffsets) throws IOException {
encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets);
}
@Override
- public void release() {
+ public synchronized void release() {
destroyImpl();
}
@@ -54,7 +55,7 @@ public class NativeXORRawEncoder extends AbstractNativeRawEncoder {
private native void encodeImpl(ByteBuffer[] inputs, int[] inputOffsets,
int dataLen, ByteBuffer[] outputs,
- int[] outputOffsets);
+ int[] outputOffsets) throws IOException;
private native void destroyImpl();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
index cfd7d29..f4257ae 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -53,7 +54,7 @@ public class RSLegacyRawDecoder extends RawErasureDecoder {
@Override
public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
- ByteBuffer[] outputs) {
+ ByteBuffer[] outputs) throws IOException {
// Make copies avoiding affecting original ones;
ByteBuffer[] newInputs = new ByteBuffer[inputs.length];
int[] newErasedIndexes = new int[erasedIndexes.length];
@@ -67,7 +68,8 @@ public class RSLegacyRawDecoder extends RawErasureDecoder {
}
@Override
- public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) {
+ public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs)
+ throws IOException {
// Make copies avoiding affecting original ones;
byte[][] newInputs = new byte[inputs.length][];
int[] newErasedIndexes = new int[erasedIndexes.length];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
index a29b472..249930e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -81,7 +82,7 @@ public abstract class RawErasureDecoder {
* erasedIndexes, ready for read after the call
*/
public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
- ByteBuffer[] outputs) {
+ ByteBuffer[] outputs) throws IOException {
ByteBufferDecodingState decodingState = new ByteBufferDecodingState(this,
inputs, erasedIndexes, outputs);
@@ -117,7 +118,8 @@ public abstract class RawErasureDecoder {
* Perform the real decoding using Direct ByteBuffer.
* @param decodingState the decoding state
*/
- protected abstract void doDecode(ByteBufferDecodingState decodingState);
+ protected abstract void doDecode(ByteBufferDecodingState decodingState)
+ throws IOException;
/**
* Decode with inputs and erasedIndexes, generates outputs. More see above.
@@ -126,8 +128,10 @@ public abstract class RawErasureDecoder {
* @param erasedIndexes indexes of erased units in the inputs array
* @param outputs output buffers to put decoded data into according to
* erasedIndexes, ready for read after the call
+ * @throws IOException if the decoder is closed.
*/
- public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) {
+ public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs)
+ throws IOException {
ByteArrayDecodingState decodingState = new ByteArrayDecodingState(this,
inputs, erasedIndexes, outputs);
@@ -142,8 +146,10 @@ public abstract class RawErasureDecoder {
* Perform the real decoding using bytes array, supporting offsets and
* lengths.
* @param decodingState the decoding state
+ * @throws IOException if the decoder is closed.
*/
- protected abstract void doDecode(ByteArrayDecodingState decodingState);
+ protected abstract void doDecode(ByteArrayDecodingState decodingState)
+ throws IOException;
/**
* Decode with inputs and erasedIndexes, generates outputs. More see above.
@@ -155,9 +161,10 @@ public abstract class RawErasureDecoder {
* @param erasedIndexes indexes of erased units in the inputs array
* @param outputs output buffers to put decoded data into according to
* erasedIndexes, ready for read after the call
+ * @throws IOException if the decoder is closed
*/
public void decode(ECChunk[] inputs, int[] erasedIndexes,
- ECChunk[] outputs) {
+ ECChunk[] outputs) throws IOException {
ByteBuffer[] newInputs = CoderUtil.toBuffers(inputs);
ByteBuffer[] newOutputs = CoderUtil.toBuffers(outputs);
decode(newInputs, erasedIndexes, newOutputs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
index 36d68f4..6d2ecd2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -62,8 +63,10 @@ public abstract class RawErasureEncoder {
* be 0 after encoding
* @param outputs output buffers to put the encoded data into, ready to read
* after the call
+ * @throws IOException if the encoder is closed.
*/
- public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs)
+ throws IOException {
ByteBufferEncodingState bbeState = new ByteBufferEncodingState(
this, inputs, outputs);
@@ -99,7 +102,8 @@ public abstract class RawErasureEncoder {
* Perform the real encoding work using direct ByteBuffer.
* @param encodingState the encoding state
*/
- protected abstract void doEncode(ByteBufferEncodingState encodingState);
+ protected abstract void doEncode(ByteBufferEncodingState encodingState)
+ throws IOException;
/**
* Encode with inputs and generates outputs. More see above.
@@ -108,7 +112,7 @@ public abstract class RawErasureEncoder {
* @param outputs output buffers to put the encoded data into, read to read
* after the call
*/
- public void encode(byte[][] inputs, byte[][] outputs) {
+ public void encode(byte[][] inputs, byte[][] outputs) throws IOException {
ByteArrayEncodingState baeState = new ByteArrayEncodingState(
this, inputs, outputs);
@@ -125,7 +129,8 @@ public abstract class RawErasureEncoder {
* and lengths.
* @param encodingState the encoding state
*/
- protected abstract void doEncode(ByteArrayEncodingState encodingState);
+ protected abstract void doEncode(ByteArrayEncodingState encodingState)
+ throws IOException;
/**
* Encode with inputs and generates outputs. More see above.
@@ -133,8 +138,9 @@ public abstract class RawErasureEncoder {
* @param inputs input buffers to read data from
* @param outputs output buffers to put the encoded data into, read to read
* after the call
+ * @throws IOException if the encoder is closed.
*/
- public void encode(ECChunk[] inputs, ECChunk[] outputs) {
+ public void encode(ECChunk[] inputs, ECChunk[] outputs) throws IOException {
ByteBuffer[] newInputs = ECChunk.toBuffers(inputs);
ByteBuffer[] newOutputs = ECChunk.toBuffers(outputs);
encode(newInputs, newOutputs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
index 2b1d9ea..9cca6dd 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
@@ -63,8 +63,9 @@ IsalCoder* getCoder(JNIEnv* env, jobject thiz) {
"Field nativeCoder not found");
}
pCoder = (IsalCoder*)(*env)->GetLongField(env, thiz, fid);
- pCoder->verbose = (verbose == JNI_TRUE) ? 1 : 0;
-
+ if (pCoder != NULL) {
+ pCoder->verbose = (verbose == JNI_TRUE) ? 1 : 0;
+ }
return pCoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c
index eb4b903..52d255a 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c
@@ -48,6 +48,10 @@ JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
jint dataLen, jintArray erasedIndexes, jobjectArray outputs,
jintArray outputOffsets) {
RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
+ if (!rsDecoder) {
+ THROW(env, "java/io/IOException", "NativeRSRawDecoder closed");
+ return;
+ }
int numDataUnits = rsDecoder->decoder.coder.numDataUnits;
int numParityUnits = rsDecoder->decoder.coder.numParityUnits;
@@ -68,5 +72,8 @@ JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl(
JNIEnv *env, jobject thiz) {
RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
- free(rsDecoder);
+ if (rsDecoder) {
+ free(rsDecoder);
+ setCoder(env, thiz, NULL);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c
index 6c477ed..a427b21 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c
@@ -47,6 +47,10 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl(
JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
jint dataLen, jobjectArray outputs, jintArray outputOffsets) {
RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
+ if (!rsEncoder) {
+ THROW(env, "java/io/IOException", "NativeRSRawEncoder closed");
+ return;
+ }
int numDataUnits = rsEncoder->encoder.coder.numDataUnits;
int numParityUnits = rsEncoder->encoder.coder.numParityUnits;
@@ -62,5 +66,8 @@ JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl(
JNIEnv *env, jobject thiz) {
RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
- free(rsEncoder);
+ if (rsEncoder) {
+ free(rsEncoder);
+ setCoder(env, thiz, NULL);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
index e8abd8b..d2de0c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
@@ -54,6 +54,10 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_decodeImpl(
XORDecoder* xorDecoder;
xorDecoder = (XORDecoder*)getCoder(env, thiz);
+ if (!xorDecoder) {
+ THROW(env, "java/io/IOException", "NativeXORRawDecoder closed");
+ return;
+ }
numDataUnits = ((IsalCoder*)xorDecoder)->numDataUnits;
numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits;
chunkSize = (int)dataLen;
@@ -76,5 +80,8 @@ JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_destroyImpl
(JNIEnv *env, jobject thiz){
XORDecoder* xorDecoder = (XORDecoder*)getCoder(env, thiz);
- free(xorDecoder);
+ if (xorDecoder) {
+ free(xorDecoder);
+ setCoder(env, thiz, NULL);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c
index 6efb5fb..ff3bf50 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c
@@ -54,6 +54,10 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder_encodeImpl(
XOREncoder* xorEncoder;
xorEncoder = (XOREncoder*)getCoder(env, thiz);
+ if (!xorEncoder) {
+ THROW(env, "java/io/IOException", "NativeXORRawEncoder closed");
+ return;
+ }
numDataUnits = ((IsalCoder*)xorEncoder)->numDataUnits;
numParityUnits = ((IsalCoder*)xorEncoder)->numParityUnits;
chunkSize = (int)dataLen;
@@ -78,5 +82,8 @@ JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder_destroyImpl
(JNIEnv *env, jobject thiz) {
XOREncoder* xorEncoder = (XOREncoder*)getCoder(env, thiz);
- free(xorEncoder);
+ if (xorEncoder) {
+ free(xorEncoder);
+ setCoder(env, thiz, NULL);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
index acbc136..753c16a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
@@ -23,8 +23,11 @@ import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.io.erasurecode.TestCoderBase;
+import java.io.IOException;
import java.lang.reflect.Constructor;
+import static org.junit.Assert.fail;
+
/**
* Erasure coder test base with utilities.
*/
@@ -85,14 +88,22 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
ErasureCodingStep codingStep;
codingStep = encoder.calculateCoding(blockGroup);
- performCodingStep(codingStep);
+ try {
+ performCodingStep(codingStep);
+ } catch (IOException e) {
+ fail("Should not expect IOException: " + e.getMessage());
+ }
// Erase specified sources but return copies of them for later comparing
TestBlock[] backupBlocks = backupAndEraseBlocks(clonedDataBlocks, parityBlocks);
// Decode
blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
codingStep = decoder.calculateCoding(blockGroup);
- performCodingStep(codingStep);
+ try {
+ performCodingStep(codingStep);
+ } catch (IOException e) {
+ fail("Should not expect IOException: " + e.getMessage());
+ }
// Compare
compareAndVerify(backupBlocks, codingStep.getOutputBlocks());
@@ -102,7 +113,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
* This is typically how a coding step should be performed.
* @param codingStep
*/
- protected void performCodingStep(ErasureCodingStep codingStep) {
+ protected void performCodingStep(ErasureCodingStep codingStep)
+ throws IOException {
// Pretend that we're opening these input blocks and output blocks.
ECBlock[] inputBlocks = codingStep.getInputBlocks();
ECBlock[] outputBlocks = codingStep.getOutputBlocks();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java
index 5471153..c27672a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHErasureCoderBase.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.io.erasurecode.ECBlock;
import org.apache.hadoop.io.erasurecode.ECChunk;
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+
/**
* Erasure coder test base with utilities for hitchhiker.
@@ -53,7 +57,11 @@ public abstract class TestHHErasureCoderBase extends TestErasureCoderBase{
}
// Given the input chunks and output chunk buffers, just call it !
- codingStep.performCoding(inputChunks, outputChunks);
+ try {
+ codingStep.performCoding(inputChunks, outputChunks);
+ } catch (IOException e) {
+ fail("Unexpected IOException: " + e.getMessage());
+ }
}
codingStep.finish();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
index bb4a1f0..c005e77 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.util.StopWatch;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.text.DecimalFormat;
import java.util.ArrayList;
@@ -232,7 +233,7 @@ public final class RawErasureCoderBenchmark {
}
}
- private static RawErasureEncoder getRawEncoder(int index) {
+ private static RawErasureEncoder getRawEncoder(int index) throws IOException {
RawErasureEncoder encoder =
CODER_MAKERS.get(index).createEncoder(BenchData.OPTIONS);
final boolean isDirect = encoder.preferDirectBuffer();
@@ -242,7 +243,7 @@ public final class RawErasureCoderBenchmark {
return encoder;
}
- private static RawErasureDecoder getRawDecoder(int index) {
+ private static RawErasureDecoder getRawDecoder(int index) throws IOException {
RawErasureDecoder decoder =
CODER_MAKERS.get(index).createDecoder(BenchData.OPTIONS);
final boolean isDirect = decoder.preferDirectBuffer();
@@ -341,11 +342,11 @@ public final class RawErasureCoderBenchmark {
System.arraycopy(inputs, 0, decodeInputs, 0, NUM_DATA_UNITS);
}
- public void encode(RawErasureEncoder encoder) {
+ public void encode(RawErasureEncoder encoder) throws IOException {
encoder.encode(inputs, outputs);
}
- public void decode(RawErasureDecoder decoder) {
+ public void decode(RawErasureDecoder decoder) throws IOException {
decoder.decode(decodeInputs, ERASED_INDEXES, outputs);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java
index 6f5871c..b936ff8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -59,7 +61,11 @@ public class TestDummyRawCoder extends TestRawCoderBase {
ECChunk[] dataChunks = prepareDataChunksForEncoding();
markChunks(dataChunks);
ECChunk[] parityChunks = prepareParityChunksForEncoding();
- encoder.encode(dataChunks, parityChunks);
+ try {
+ encoder.encode(dataChunks, parityChunks);
+ } catch (IOException e) {
+ Assert.fail("Unexpected IOException: " + e.getMessage());
+ }
compareAndVerify(parityChunks, getEmptyChunks(parityChunks.length));
// Decode
@@ -69,7 +75,12 @@ public class TestDummyRawCoder extends TestRawCoderBase {
dataChunks, parityChunks);
ensureOnlyLeastRequiredChunks(inputChunks);
ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
- decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+ try {
+ decoder.decode(inputChunks, getErasedIndexesForDecoding(),
+ recoveredChunks);
+ } catch (IOException e) {
+ Assert.fail("Unexpected IOException: " + e.getMessage());
+ }
compareAndVerify(recoveredChunks, getEmptyChunks(recoveredChunks.length));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java
index e3536d8..d56045e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java
@@ -118,4 +118,10 @@ public class TestNativeRSRawCoder extends TestRSRawCoderBase {
prepare(null, 10, 4, new int[] {0}, new int[] {0});
testCodingDoMixAndTwice();
}
+
+ @Test
+ public void testAfterRelease63() throws Exception {
+ prepare(6, 3, null, null);
+ testAfterRelease();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java
index 5adefbe..90e9410 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.junit.Assume;
import org.junit.Before;
+import org.junit.Test;
/**
* Test NativeXOR encoding and decoding.
@@ -33,4 +34,10 @@ public class TestNativeXORRawCoder extends TestXORRawCoderBase {
this.decoderFactoryClass = NativeXORRawErasureCoderFactory.class;
setAllowDump(true);
}
+
+ @Test
+ public void testAfterRelease63() throws Exception {
+ prepare(6, 3, null, null);
+ testAfterRelease();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
index 01c743a..4519e35 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
@@ -20,9 +20,12 @@ package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.io.erasurecode.TestCoderBase;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
+
/**
* Raw coder test base with utilities.
*/
@@ -104,6 +107,28 @@ public abstract class TestRawCoderBase extends TestCoderBase {
}
}
+ /**
+ * Test encode / decode after release(). It should raise IOException.
+ *
+ * @throws Exception
+ */
+ void testAfterRelease() throws Exception {
+ prepareCoders(true);
+ prepareBufferAllocator(true);
+
+ encoder.release();
+ final ECChunk[] data = prepareDataChunksForEncoding();
+ final ECChunk[] parity = prepareParityChunksForEncoding();
+ LambdaTestUtils.intercept(IOException.class, "closed",
+ () -> encoder.encode(data, parity));
+
+ decoder.release();
+ final ECChunk[] in = prepareInputChunksForDecoding(data, parity);
+ final ECChunk[] out = prepareOutputChunksForDecoding();
+ LambdaTestUtils.intercept(IOException.class, "closed",
+ () -> decoder.decode(in, getErasedIndexesForDecoding(), out));
+ }
+
@Test
public void testCodingWithErasingTooMany() {
try {
@@ -121,6 +146,16 @@ public abstract class TestRawCoderBase extends TestCoderBase {
}
}
+ @Test
+ public void testIdempotentReleases() {
+ prepareCoders(true);
+
+ for (int i = 0; i < 3; i++) {
+ encoder.release();
+ decoder.release();
+ }
+ }
+
private void performTestCoding(int chunkSize, boolean usingSlicedBuffer,
boolean useBadInput, boolean useBadOutput,
boolean allowChangeInputs) {
@@ -144,7 +179,11 @@ public abstract class TestRawCoderBase extends TestCoderBase {
ECChunk[] clonedDataChunks = cloneChunksWithData(dataChunks);
markChunks(dataChunks);
- encoder.encode(dataChunks, parityChunks);
+ try {
+ encoder.encode(dataChunks, parityChunks);
+ } catch (IOException e) {
+ Assert.fail("Should not get IOException: " + e.getMessage());
+ }
dumpChunks("Encoded parity chunks", parityChunks);
if (!allowChangeInputs) {
@@ -174,7 +213,12 @@ public abstract class TestRawCoderBase extends TestCoderBase {
}
dumpChunks("Decoding input chunks", inputChunks);
- decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+ try {
+ decoder.decode(inputChunks, getErasedIndexesForDecoding(),
+ recoveredChunks);
+ } catch (IOException e) {
+ Assert.fail("Should not get IOException: " + e.getMessage());
+ }
dumpChunks("Decoded/recovered chunks", recoveredChunks);
if (!allowChangeInputs) {
@@ -268,7 +312,11 @@ public abstract class TestRawCoderBase extends TestCoderBase {
ECChunk[] dataChunks = prepareDataChunksForEncoding();
ECChunk[] parityChunks = prepareParityChunksForEncoding();
ECChunk[] clonedDataChunks = cloneChunksWithData(dataChunks);
- encoder.encode(dataChunks, parityChunks);
+ try {
+ encoder.encode(dataChunks, parityChunks);
+ } catch (IOException e) {
+ Assert.fail("Should not get IOException: " + e.getMessage());
+ }
verifyBufferPositionAtEnd(dataChunks);
// verify decode
@@ -277,7 +325,12 @@ public abstract class TestRawCoderBase extends TestCoderBase {
clonedDataChunks, parityChunks);
ensureOnlyLeastRequiredChunks(inputChunks);
ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
- decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+ try {
+ decoder.decode(inputChunks, getErasedIndexesForDecoding(),
+ recoveredChunks);
+ } catch (IOException e) {
+ Assert.fail("Should not get IOException: " + e.getMessage());
+ }
verifyBufferPositionAtEnd(inputChunks);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 3eb7e67..1b83959 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -355,7 +355,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
* @param buffers data buffers + parity buffers
*/
private static void encode(RawErasureEncoder encoder, int numData,
- ByteBuffer[] buffers) {
+ ByteBuffer[] buffers) throws IOException {
final ByteBuffer[] dataBuffers = new ByteBuffer[numData];
final ByteBuffer[] parityBuffers = new ByteBuffer[buffers.length - numData];
System.arraycopy(buffers, 0, dataBuffers, 0, dataBuffers.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
index b01b74c..65d2c90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -68,7 +69,7 @@ class PositionStripeReader extends StripeReader {
}
@Override
- void decode() {
+ void decode() throws IOException {
finalizeDecodeInputs();
decodeAndFillBuffer(true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
index 8879514..b37501d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
+import java.io.IOException;
import java.nio.ByteBuffer;
/**
@@ -88,7 +89,7 @@ class StatefulStripeReader extends StripeReader {
}
@Override
- void decode() {
+ void decode() throws IOException {
finalizeDecodeInputs();
decodeAndFillBuffer(false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
index 5518752..9a20423 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
@@ -149,10 +149,11 @@ abstract class StripeReader {
*/
abstract boolean prepareParityChunk(int index);
- /*
+ /**
* Decode to get the missing data.
+ * @throws IOException if the decoder is closed.
*/
- abstract void decode();
+ abstract void decode() throws IOException;
/*
* Default close do nothing.
@@ -408,7 +409,7 @@ abstract class StripeReader {
/**
* Decode based on the given input buffers and erasure coding policy.
*/
- void decodeAndFillBuffer(boolean fillBuffer) {
+ void decodeAndFillBuffer(boolean fillBuffer) throws IOException {
// Step 1: prepare indices and output buffers for missing data units
int[] decodeIndices = prepareErasedIndices();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
index 6917317..6e063c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
@@ -155,7 +155,7 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
return checksumBuf.length;
}
- private void reconstructTargets(int toReconstructLen) {
+ private void reconstructTargets(int toReconstructLen) throws IOException {
ByteBuffer[] inputs = getStripedReader().getInputBuffers(toReconstructLen);
ByteBuffer[] outputs = new ByteBuffer[1];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index 34e58ae..17b54e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -113,7 +113,7 @@ class StripedBlockReconstructor extends StripedReconstructor
}
}
- private void reconstructTargets(int toReconstructLen) {
+ private void reconstructTargets(int toReconstructLen) throws IOException {
ByteBuffer[] inputs = getStripedReader().getInputBuffers(toReconstructLen);
int[] erasedIndices = stripedWriter.getRealTargetIndices();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ebccc9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
index c771d21..08bf20a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
@@ -502,7 +502,11 @@ public class StripedFileTestUtil {
dataBytes.length, parityBytes.length);
final RawErasureEncoder encoder =
CodecUtil.createRawEncoder(conf, codecName, coderOptions);
- encoder.encode(dataBytes, expectedParityBytes);
+ try {
+ encoder.encode(dataBytes, expectedParityBytes);
+ } catch (IOException e) {
+ Assert.fail("Unexpected IOException: " + e.getMessage());
+ }
for (int i = 0; i < parityBytes.length; i++) {
if (checkSet.contains(i + dataBytes.length)){
Assert.assertArrayEquals("i=" + i, expectedParityBytes[i],
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] hadoop git commit: YARN-7275. NM Statestore cleanup for
Container updates. (Kartheek Muthyala via asuresh)
Posted by ae...@apache.org.
YARN-7275. NM Statestore cleanup for Container updates. (Kartheek Muthyala via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a50be1b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a50be1b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a50be1b8
Branch: refs/heads/HDFS-7240
Commit: a50be1b8f432f50c940b66d12c7de87b95ea47c0
Parents: 8dbc890
Author: Arun Suresh <as...@apache.org>
Authored: Mon Oct 16 13:08:52 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Mon Oct 16 13:12:15 2017 -0700
----------------------------------------------------------------------
.../containermanager/ContainerManagerImpl.java | 9 +++
.../container/ContainerEventType.java | 4 +-
.../container/ContainerImpl.java | 43 +++++++++-----
.../launcher/ContainersLauncher.java | 2 -
.../launcher/RecoverPausedContainerLaunch.java | 38 +++++++-----
.../launcher/RecoveredContainerLaunch.java | 2 +-
.../scheduler/ContainerScheduler.java | 31 ++++++++++
.../scheduler/ContainerSchedulerEventType.java | 3 +-
.../recovery/NMLeveldbStateStoreService.java | 62 ++++++++++++++------
.../recovery/NMNullStateStoreService.java | 6 +-
.../recovery/NMStateStoreService.java | 10 ++--
.../TestContainerManagerRecovery.java | 4 ++
.../recovery/NMMemoryStateStoreService.java | 18 ++++--
.../TestNMLeveldbStateStoreService.java | 16 +++--
14 files changed, 179 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 38eb636..7d5525a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -367,6 +368,13 @@ public class ContainerManagerImpl extends CompositeService implements
}
recoverContainer(rcs);
}
+
+ //Dispatching the RECOVERY_COMPLETED event through the dispatcher
+ //so that all the paused, scheduled and queued containers will
+ //be scheduled for execution on availability of resources.
+ dispatcher.getEventHandler().handle(
+ new ContainerSchedulerEvent(null,
+ ContainerSchedulerEventType.RECOVERY_COMPLETED));
} else {
LOG.info("Not a recoverable state store. Nothing to recover.");
}
@@ -480,6 +488,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container container = new ContainerImpl(getConfig(), dispatcher,
launchContext, credentials, metrics, token, context, rcs);
context.getContainers().put(token.getContainerID(), container);
+ containerScheduler.recoverActiveContainer(container, rcs.getStatus());
app.handle(new ApplicationContainerInitEvent(container));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
index e28b37d..75e32e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
@@ -46,5 +46,7 @@ public enum ContainerEventType {
CONTAINER_RESUMED,
// Producer: ContainerScheduler
- CONTAINER_TOKEN_UPDATED
+ CONTAINER_TOKEN_UPDATED,
+
+ RECOVER_PAUSED_CONTAINER
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index df83789..17b24b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -362,6 +362,9 @@ public class ContainerImpl implements Container {
// From SCHEDULED State
.addTransition(ContainerState.SCHEDULED, ContainerState.RUNNING,
ContainerEventType.CONTAINER_LAUNCHED, new LaunchTransition())
+ .addTransition(ContainerState.SCHEDULED, ContainerState.PAUSED,
+ ContainerEventType.RECOVER_PAUSED_CONTAINER,
+ new RecoveredContainerTransition())
.addTransition(ContainerState.SCHEDULED, ContainerState.EXITED_WITH_FAILURE,
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
new ExitedWithFailureTransition(true))
@@ -952,7 +955,10 @@ public class ContainerImpl implements Container {
if (recoveredStatus == RecoveredContainerStatus.LAUNCHED) {
// try to recover a container that was previously launched
launcherEvent = ContainersLauncherEventType.RECOVER_CONTAINER;
+ } else if (recoveredStatus == RecoveredContainerStatus.PAUSED) {
+ launcherEvent = ContainersLauncherEventType.RECOVER_PAUSED_CONTAINER;
}
+
containerLaunchStartTime = clock.getTime();
dispatcher.getEventHandler().handle(
new ContainersLauncherEvent(this, launcherEvent));
@@ -963,9 +969,6 @@ public class ContainerImpl implements Container {
@SuppressWarnings("unchecked") // dispatcher not typed
private void sendScheduleEvent() {
if (recoveredStatus == RecoveredContainerStatus.PAUSED) {
- // Recovery is not supported for paused container so we raise the
- // launch event which will proceed to kill the paused container instead
- // of raising the schedule event.
ContainersLauncherEventType launcherEvent;
launcherEvent = ContainersLauncherEventType.RECOVER_PAUSED_CONTAINER;
dispatcher.getEventHandler()
@@ -1060,17 +1063,15 @@ public class ContainerImpl implements Container {
UpdateContainerTokenEvent updateEvent = (UpdateContainerTokenEvent)event;
// Update the container token
container.setContainerTokenIdentifier(updateEvent.getUpdatedToken());
- if (updateEvent.isResourceChange()) {
- try {
- // Persist change in the state store.
- container.context.getNMStateStore().storeContainerResourceChanged(
- container.containerId,
- container.getContainerTokenIdentifier().getVersion(),
- container.getResource());
- } catch (IOException e) {
- LOG.warn("Could not store container [" + container.containerId
- + "] resource change..", e);
- }
+
+ try {
+ // Persist change in the state store.
+ container.context.getNMStateStore()
+ .storeContainerUpdateToken(container.containerId,
+ container.getContainerTokenIdentifier());
+ } catch (IOException e) {
+ LOG.warn("Could not store container [" + container.containerId
+ + "] update..", e);
}
}
}
@@ -1115,6 +1116,8 @@ public class ContainerImpl implements Container {
if (container.recoveredStatus == RecoveredContainerStatus.COMPLETED) {
container.sendFinishedEvents();
return ContainerState.DONE;
+ } else if (container.recoveredStatus == RecoveredContainerStatus.QUEUED) {
+ return ContainerState.SCHEDULED;
} else if (container.recoveredAsKilled &&
container.recoveredStatus == RecoveredContainerStatus.REQUESTED) {
// container was killed but never launched
@@ -1474,6 +1477,18 @@ public class ContainerImpl implements Container {
}
/**
+ * Transition from SCHEDULED state to PAUSED state on recovery
+ */
+ static class RecoveredContainerTransition extends ContainerTransition {
+ @SuppressWarnings("unchecked")
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ container.sendContainerMonitorStartEvent();
+ container.wasLaunched = true;
+ }
+ }
+
+ /**
* Transition from RUNNING or KILLING state to
* EXITED_WITH_SUCCESS state upon EXITED_WITH_SUCCESS message.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 9f6ef74..cfd5d6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -140,8 +140,6 @@ public class ContainersLauncher extends AbstractService
running.put(containerId, launch);
break;
case RECOVER_PAUSED_CONTAINER:
- // Recovery for paused containers is not supported, thus here
- // we locate any paused containers, and terminate them.
app = context.getApplications().get(
containerId.getApplicationAttemptId().getApplicationId());
launch = new RecoverPausedContainerLaunch(context, getConfig(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoverPausedContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoverPausedContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoverPausedContainerLaunch.java
index 14cab9a..761fe3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoverPausedContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoverPausedContainerLaunch.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.*;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
import java.io.File;
import java.io.IOException;
@@ -66,6 +66,8 @@ public class RecoverPausedContainerLaunch extends ContainerLaunch {
containerId.getApplicationAttemptId().getApplicationId().toString();
String containerIdStr = containerId.toString();
+ dispatcher.getEventHandler().handle(new ContainerEvent(containerId,
+ ContainerEventType.RECOVER_PAUSED_CONTAINER));
boolean notInterrupted = true;
try {
File pidFile = locatePidFile(appIdStr, containerIdStr);
@@ -73,16 +75,17 @@ public class RecoverPausedContainerLaunch extends ContainerLaunch {
String pidPathStr = pidFile.getPath();
pidFilePath = new Path(pidPathStr);
exec.activateContainer(containerId, pidFilePath);
- exec.signalContainer(new ContainerSignalContext.Builder()
- .setContainer(container)
- .setUser(container.getUser())
- .setSignal(ContainerExecutor.Signal.KILL)
- .build());
+ retCode = exec.reacquireContainer(
+ new ContainerReacquisitionContext.Builder()
+ .setContainer(container)
+ .setUser(container.getUser())
+ .setContainerId(containerId)
+ .build());
} else {
LOG.warn("Unable to locate pid file for container " + containerIdStr);
}
- } catch (InterruptedIOException e) {
+ } catch (InterruptedException | InterruptedIOException e) {
LOG.warn("Interrupted while waiting for exit code from " + containerId);
notInterrupted = false;
} catch (IOException e) {
@@ -100,14 +103,21 @@ public class RecoverPausedContainerLaunch extends ContainerLaunch {
}
}
- LOG.warn("Recovered container exited with a non-zero exit code "
- + retCode);
- this.dispatcher.getEventHandler().handle(new ContainerExitEvent(
- containerId,
- ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, retCode,
- "Container exited with a non-zero exit code " + retCode));
+ if (retCode != 0) {
+ LOG.warn("Recovered container exited with a non-zero exit code "
+ + retCode);
+ this.dispatcher.getEventHandler().handle(new ContainerExitEvent(
+ containerId,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, retCode,
+ "Container exited with a non-zero exit code " + retCode));
+ return retCode;
+ }
- return retCode;
+ LOG.info("Recovered container " + containerId + " succeeded");
+ dispatcher.getEventHandler().handle(
+ new ContainerEvent(containerId,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS));
+ return 0;
}
private File locatePidFile(String appIdStr, String containerIdStr) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
index 17ddd77..a3ccf00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
@@ -72,7 +72,7 @@ public class RecoveredContainerLaunch extends ContainerLaunch {
String containerIdStr = containerId.toString();
dispatcher.getEventHandler().handle(new ContainerEvent(containerId,
- ContainerEventType.CONTAINER_LAUNCHED));
+ ContainerEventType.RECOVER_PAUSED_CONTAINER));
boolean notInterrupted = true;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index e436822..76da37c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -168,6 +169,8 @@ public class ContainerScheduler extends AbstractService implements
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
break;
+ case RECOVERY_COMPLETED:
+ startPendingContainers(maxOppQueueLength <= 0);
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
@@ -219,6 +222,34 @@ public class ContainerScheduler extends AbstractService implements
}
/**
+ * Populates auxiliary data structures used by the ContainerScheduler on
+ * recovery.
+ * @param container container recovered
+ * @param rcs Recovered Container status
+ */
+ public void recoverActiveContainer(Container container,
+ RecoveredContainerStatus rcs) {
+ ExecutionType execType =
+ container.getContainerTokenIdentifier().getExecutionType();
+ if (rcs == RecoveredContainerStatus.QUEUED
+ || rcs == RecoveredContainerStatus.PAUSED) {
+ if (execType == ExecutionType.GUARANTEED) {
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
+ } else if (execType == ExecutionType.OPPORTUNISTIC) {
+ queuedOpportunisticContainers
+ .put(container.getContainerId(), container);
+ } else {
+ LOG.error(
+ "UnKnown execution type received " + container.getContainerId()
+ + ", execType " + execType);
+ }
+ } else if (rcs == RecoveredContainerStatus.LAUNCHED) {
+ runningContainers.put(container.getContainerId(), container);
+ utilizationTracker.addContainerResources(container);
+ }
+ }
+
+ /**
* Return number of queued containers.
* @return Number of queued containers.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index a9cbf74..294eddf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -27,5 +27,6 @@ public enum ContainerSchedulerEventType {
UPDATE_CONTAINER,
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
- CONTAINER_PAUSED
+ CONTAINER_PAUSED,
+ RECOVERY_COMPLETED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 2f9c0a7..c361d00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager.recovery;
import static org.fusesource.leveldbjni.JniDBFactory.asString;
import static org.fusesource.leveldbjni.JniDBFactory.bytes;
+
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.slf4j.LoggerFactory;
import java.io.File;
@@ -47,11 +50,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestP
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.Deletion
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto;
+import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
@@ -120,8 +121,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
private static final String CONTAINER_LAUNCHED_KEY_SUFFIX = "/launched";
private static final String CONTAINER_QUEUED_KEY_SUFFIX = "/queued";
private static final String CONTAINER_PAUSED_KEY_SUFFIX = "/paused";
- private static final String CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX =
- "/resourceChanged";
+ private static final String CONTAINER_UPDATE_TOKEN_SUFFIX =
+ "/updateToken";
private static final String CONTAINER_KILLED_KEY_SUFFIX = "/killed";
private static final String CONTAINER_EXIT_CODE_KEY_SUFFIX = "/exitcode";
private static final String CONTAINER_REMAIN_RETRIES_KEY_SUFFIX =
@@ -290,9 +291,17 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
} else if (suffix.equals(CONTAINER_EXIT_CODE_KEY_SUFFIX)) {
rcs.status = RecoveredContainerStatus.COMPLETED;
rcs.exitCode = Integer.parseInt(asString(entry.getValue()));
- } else if (suffix.equals(CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX)) {
- rcs.capability = new ResourcePBImpl(
- ResourceProto.parseFrom(entry.getValue()));
+ } else if (suffix.equals(CONTAINER_UPDATE_TOKEN_SUFFIX)) {
+ ContainerTokenIdentifierProto tokenIdentifierProto =
+ ContainerTokenIdentifierProto.parseFrom(entry.getValue());
+ Token currentToken = rcs.getStartRequest().getContainerToken();
+ Token updatedToken = Token
+ .newInstance(tokenIdentifierProto.toByteArray(),
+ ContainerTokenIdentifier.KIND.toString(),
+ currentToken.getPassword().array(), currentToken.getService());
+ rcs.startRequest.setContainerToken(updatedToken);
+ rcs.capability = new ResourcePBImpl(tokenIdentifierProto.getResource());
+ rcs.version = tokenIdentifierProto.getVersion();
} else if (suffix.equals(CONTAINER_REMAIN_RETRIES_KEY_SUFFIX)) {
rcs.setRemainingRetryAttempts(
Integer.parseInt(asString(entry.getValue())));
@@ -374,6 +383,21 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
}
}
+ private void removeContainerQueued(ContainerId containerId)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("removeContainerQueued: containerId=" + containerId);
+ }
+
+ String key = CONTAINERS_KEY_PREFIX + containerId.toString()
+ + CONTAINER_QUEUED_KEY_SUFFIX;
+ try {
+ db.delete(bytes(key));
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ }
+
@Override
public void storeContainerPaused(ContainerId containerId) throws IOException {
if (LOG.isDebugEnabled()) {
@@ -429,6 +453,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
LOG.debug("storeContainerLaunched: containerId=" + containerId);
}
+ // Removing the container if queued for backward compatibility reasons
+ removeContainerQueued(containerId);
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
+ CONTAINER_LAUNCHED_KEY_SUFFIX;
try {
@@ -439,24 +465,25 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
}
@Override
- public void storeContainerResourceChanged(ContainerId containerId,
- int containerVersion, Resource capability) throws IOException {
+ public void storeContainerUpdateToken(ContainerId containerId,
+ ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
if (LOG.isDebugEnabled()) {
- LOG.debug("storeContainerResourceChanged: containerId=" + containerId
- + ", capability=" + capability);
+ LOG.debug("storeContainerUpdateToken: containerId=" + containerId);
}
- String keyResChng = CONTAINERS_KEY_PREFIX + containerId.toString()
- + CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX;
+ String keyUpdateToken = CONTAINERS_KEY_PREFIX + containerId.toString()
+ + CONTAINER_UPDATE_TOKEN_SUFFIX;
String keyVersion = CONTAINERS_KEY_PREFIX + containerId.toString()
+ CONTAINER_VERSION_KEY_SUFFIX;
+
try {
WriteBatch batch = db.createWriteBatch();
try {
// New value will overwrite old values for the same key
- batch.put(bytes(keyResChng),
- ProtoUtils.convertToProtoFormat(capability).toByteArray());
- batch.put(bytes(keyVersion), bytes(Integer.toString(containerVersion)));
+ batch.put(bytes(keyUpdateToken),
+ containerTokenIdentifier.getProto().toByteArray());
+ batch.put(bytes(keyVersion),
+ bytes(Integer.toString(containerTokenIdentifier.getVersion())));
db.write(batch);
} finally {
batch.close();
@@ -552,6 +579,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
batch.delete(bytes(keyPrefix + CONTAINER_PAUSED_KEY_SUFFIX));
batch.delete(bytes(keyPrefix + CONTAINER_KILLED_KEY_SUFFIX));
batch.delete(bytes(keyPrefix + CONTAINER_EXIT_CODE_KEY_SUFFIX));
+ batch.delete(bytes(keyPrefix + CONTAINER_UPDATE_TOKEN_SUFFIX));
List<String> unknownKeysForContainer = containerUnknownKeySuffixes
.removeAll(containerId);
for (String unknownKeySuffix : unknownKeysForContainer) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
index d1d0696..ca6d018 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
@@ -28,12 +28,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
// The state store to use when state isn't being stored
@@ -99,8 +99,8 @@ public class NMNullStateStoreService extends NMStateStoreService {
}
@Override
- public void storeContainerResourceChanged(ContainerId containerId,
- int version, Resource capability) throws IOException {
+ public void storeContainerUpdateToken(ContainerId containerId,
+ ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
index 999d2d9..5e2b8a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.Containe
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
@@ -429,14 +430,13 @@ public abstract class NMStateStoreService extends AbstractService {
throws IOException;
/**
- * Record that a container resource has been changed
+ * Record that a container has been updated
* @param containerId the container ID
- * @param containerVersion the container version
- * @param capability the container resource capability
+ * @param containerTokenIdentifier container token identifier
* @throws IOException
*/
- public abstract void storeContainerResourceChanged(ContainerId containerId,
- int containerVersion, Resource capability) throws IOException;
+ public abstract void storeContainerUpdateToken(ContainerId containerId,
+ ContainerTokenIdentifier containerTokenIdentifier) throws IOException;
/**
* Record that a container has completed
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 5ec0ae6..0e629d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -405,6 +405,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
stateStore.start();
context = createContext(conf, stateStore);
ContainerManagerImpl cm = createContainerManager(context, delSrvc);
+ ((NMContext) context).setContainerManager(cm);
cm.init(conf);
cm.start();
// add an application by starting a container
@@ -430,6 +431,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
cm.stop();
context = createContext(conf, stateStore);
cm = createContainerManager(context);
+ ((NMContext) context).setContainerManager(cm);
cm.init(conf);
cm.start();
assertEquals(1, context.getApplications().size());
@@ -448,6 +450,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
stateStore.start();
context = createContext(conf, stateStore);
ContainerManagerImpl cm = createContainerManager(context, delSrvc);
+ ((NMContext) context).setContainerManager(cm);
cm.init(conf);
cm.start();
@@ -473,6 +476,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
cm.stop();
context = createContext(conf, stateStore);
cm = createContainerManager(context);
+ ((NMContext) context).setContainerManager(cm);
cm.init(conf);
cm.start();
assertEquals(1, context.getApplications().size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
index 59a225a..eb222cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
@@ -175,12 +176,17 @@ public class NMMemoryStateStoreService extends NMStateStoreService {
}
@Override
- public synchronized void storeContainerResourceChanged(
- ContainerId containerId, int version, Resource capability)
- throws IOException {
+ public void storeContainerUpdateToken(ContainerId containerId,
+ ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
RecoveredContainerState rcs = getRecoveredContainerState(containerId);
- rcs.capability = capability;
- rcs.version = version;
+ rcs.capability = containerTokenIdentifier.getResource();
+ rcs.version = containerTokenIdentifier.getVersion();
+ Token currentToken = rcs.getStartRequest().getContainerToken();
+ Token updatedToken = Token
+ .newInstance(containerTokenIdentifier.getBytes(),
+ ContainerTokenIdentifier.KIND.toString(),
+ currentToken.getPassword().array(), currentToken.getService());
+ rcs.startRequest.setContainerToken(updatedToken);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a50be1b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 8c13356..1ff2119 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -307,13 +307,18 @@ public class TestNMLeveldbStateStoreService {
assertEquals(1, recoveredContainers.size());
// increase the container size, and verify recovered
- stateStore.storeContainerResourceChanged(containerId, 2,
- Resource.newInstance(2468, 4));
+ ContainerTokenIdentifier updateTokenIdentifier =
+ new ContainerTokenIdentifier(containerId, "host", "user",
+ Resource.newInstance(2468, 4), 9876543210L, 42, 2468,
+ Priority.newInstance(7), 13579);
+
+ stateStore
+ .storeContainerUpdateToken(containerId, updateTokenIdentifier);
restartStateStore();
recoveredContainers = stateStore.loadContainersState();
assertEquals(1, recoveredContainers.size());
rcs = recoveredContainers.get(0);
- assertEquals(2, rcs.getVersion());
+ assertEquals(0, rcs.getVersion());
assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus());
assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
assertEquals(false, rcs.getKilled());
@@ -330,7 +335,9 @@ public class TestNMLeveldbStateStoreService {
assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus());
assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
assertTrue(rcs.getKilled());
- assertEquals(containerReq, rcs.getStartRequest());
+ ContainerTokenIdentifier tokenReadFromRequest = BuilderUtils
+ .newContainerTokenIdentifier(rcs.getStartRequest().getContainerToken());
+ assertEquals(updateTokenIdentifier, tokenReadFromRequest);
assertEquals(diags.toString(), rcs.getDiagnostics());
// add yet more diags, mark container completed, and verify recovered
@@ -344,7 +351,6 @@ public class TestNMLeveldbStateStoreService {
assertEquals(RecoveredContainerStatus.COMPLETED, rcs.getStatus());
assertEquals(21, rcs.getExitCode());
assertTrue(rcs.getKilled());
- assertEquals(containerReq, rcs.getStartRequest());
assertEquals(diags.toString(), rcs.getDiagnostics());
// store remainingRetryAttempts, workDir and logDir
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] hadoop git commit: YARN-7294.
TestSignalContainer#testSignalRequestDeliveryToNM fails intermittently with
Fair Scheduler. (Contributed by Miklos Szegedi)
Posted by ae...@apache.org.
YARN-7294. TestSignalContainer#testSignalRequestDeliveryToNM fails intermittently with Fair Scheduler. (Contributed by Miklos Szegedi)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbd2b73e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbd2b73e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbd2b73e
Branch: refs/heads/HDFS-7240
Commit: cbd2b73ef81a7e275c5d4f842cac5b81ff2f8c84
Parents: c1b08ba
Author: Yufei Gu <yu...@apache.org>
Authored: Thu Oct 19 16:39:25 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Thu Oct 19 16:39:25 2017 -0700
----------------------------------------------------------------------
.../yarn/server/resourcemanager/TestSignalContainer.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbd2b73e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java
index 2688987..fac0b96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSignalContainer.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -50,6 +51,10 @@ public class TestSignalContainer {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
MockRM rm = new MockRM();
+ FairScheduler fs = null;
+ if (rm.getResourceScheduler().getClass() == FairScheduler.class) {
+ fs = (FairScheduler)rm.getResourceScheduler();
+ }
rm.start();
MockNM nm1 = rm.registerNode("h1:1234", 5000);
@@ -78,6 +83,9 @@ public class TestSignalContainer {
List<Container> allocation = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers();
conts.addAll(allocation);
+ if (fs != null) {
+ nm1.nodeHeartbeat(true);
+ }
}
Assert.assertEquals(request, conts.size());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] hadoop git commit: HADOOP-14944. Add JvmMetrics to KMS.
Posted by ae...@apache.org.
HADOOP-14944. Add JvmMetrics to KMS.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86ee0c5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86ee0c5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86ee0c5e
Branch: refs/heads/HDFS-7240
Commit: 86ee0c5e4e304d9551a24a3d8b9161ca1502b70e
Parents: 46eb103
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Oct 17 15:55:30 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Oct 17 19:06:45 2017 -0700
----------------------------------------------------------------------
.../hadoop/metrics2/source/JvmMetrics.java | 16 +++++++++
.../crypto/key/kms/server/KMSConfiguration.java | 9 +++++
.../crypto/key/kms/server/KMSWebServer.java | 24 ++++++++++++++
.../hadoop/crypto/key/kms/server/TestKMS.java | 35 ++++++++++++++++++++
4 files changed, 84 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ee0c5e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index c6369cd..e3f8754 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -58,6 +58,11 @@ public class JvmMetrics implements MetricsSource {
}
return impl;
}
+
+ synchronized void shutdown() {
+ DefaultMetricsSystem.instance().unregisterSource(JvmMetrics.name());
+ impl = null;
+ }
}
@VisibleForTesting
@@ -81,6 +86,7 @@ public class JvmMetrics implements MetricsSource {
final ConcurrentHashMap<String, MetricsInfo[]> gcInfoCache =
new ConcurrentHashMap<String, MetricsInfo[]>();
+ @VisibleForTesting
JvmMetrics(String processName, String sessionId) {
this.processName = processName;
this.sessionId = sessionId;
@@ -104,6 +110,16 @@ public class JvmMetrics implements MetricsSource {
return Singleton.INSTANCE.init(processName, sessionId);
}
+ /**
+ * Shutdown the JvmMetrics singleton. This is not necessary if the JVM itself
+ * is shutdown, but may be necessary for scenarios where JvmMetrics instance
+ * needs to be re-created while the JVM is still around. One such scenario
+ * is unit-testing.
+ */
+ public static void shutdownSingleton() {
+ Singleton.INSTANCE.shutdown();
+ }
+
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
MetricsRecordBuilder rb = collector.addRecord(JvmMetrics)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ee0c5e/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index df17ef5..18eec19 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -76,6 +76,15 @@ public class KMSConfiguration {
public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX +
"audit.aggregation.window.ms";
+ // Process name shown in metrics
+ public static final String METRICS_PROCESS_NAME_KEY =
+ CONFIG_PREFIX + "metrics.process.name";
+ public static final String METRICS_PROCESS_NAME_DEFAULT = "KMS";
+
+ // Session id for metrics
+ public static final String METRICS_SESSION_ID_KEY =
+ CONFIG_PREFIX + "metrics.session.id";
+
// KMS Audit logger classes to use
public static final String KMS_AUDIT_LOGGER_KEY = CONFIG_PREFIX +
"audit.logger";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ee0c5e/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
index ced1f69..19f7227 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
@@ -27,12 +27,19 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.ConfigurationWithLogging;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_PROCESS_NAME_DEFAULT;
+import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_PROCESS_NAME_KEY;
+import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.METRICS_SESSION_ID_KEY;
+
/**
* The KMS web server.
*/
@@ -46,6 +53,9 @@ public class KMSWebServer {
private final HttpServer2 httpServer;
private final String scheme;
+ private final String processName;
+ private final String sessionId;
+ private final JvmPauseMonitor pauseMonitor;
KMSWebServer(Configuration conf, Configuration sslConf) throws Exception {
// Override configuration with deprecated environment variables.
@@ -73,6 +83,11 @@ public class KMSWebServer {
boolean sslEnabled = conf.getBoolean(KMSConfiguration.SSL_ENABLED_KEY,
KMSConfiguration.SSL_ENABLED_DEFAULT);
scheme = sslEnabled ? HttpServer2.HTTPS_SCHEME : HttpServer2.HTTP_SCHEME;
+ processName =
+ conf.get(METRICS_PROCESS_NAME_KEY, METRICS_PROCESS_NAME_DEFAULT);
+ sessionId = conf.get(METRICS_SESSION_ID_KEY);
+ pauseMonitor = new JvmPauseMonitor();
+ pauseMonitor.init(conf);
String host = conf.get(KMSConfiguration.HTTP_HOST_KEY,
KMSConfiguration.HTTP_HOST_DEFAULT);
@@ -113,6 +128,11 @@ public class KMSWebServer {
public void start() throws IOException {
httpServer.start();
+
+ DefaultMetricsSystem.initialize(processName);
+ final JvmMetrics jm = JvmMetrics.initSingleton(processName, sessionId);
+ jm.setPauseMonitor(pauseMonitor);
+ pauseMonitor.start();
}
public boolean isRunning() {
@@ -125,6 +145,10 @@ public class KMSWebServer {
public void stop() throws Exception {
httpServer.stop();
+
+ pauseMonitor.stop();
+ JvmMetrics.shutdownSingleton();
+ DefaultMetricsSystem.shutdown();
}
public URL getKMSUrl() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ee0c5e/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 30e84eb..7605b28 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -98,6 +98,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
@@ -2693,4 +2694,38 @@ public class TestKMS {
});
}
+ /*
+ * Test the jmx page can return, and contains the basic JvmMetrics. Only
+ * testing in simple mode since the page content is the same, kerberized
+ * or not.
+ */
+ @Test
+ public void testKMSJMX() throws Exception {
+ Configuration conf = new Configuration();
+ final File confDir = getTestDir();
+ conf = createBaseKMSConf(confDir, conf);
+ final String processName = "testkmsjmx";
+ conf.set(KMSConfiguration.METRICS_PROCESS_NAME_KEY, processName);
+ writeConf(confDir, conf);
+
+ runServer(null, null, confDir, new KMSCallable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final URL jmxUrl = new URL(
+ getKMSUrl() + "/jmx?user.name=whatever&qry=Hadoop:service="
+ + processName + ",name=JvmMetrics");
+ LOG.info("Requesting jmx from " + jmxUrl);
+ final StringBuilder sb = new StringBuilder();
+ final InputStream in = jmxUrl.openConnection().getInputStream();
+ final byte[] buffer = new byte[64 * 1024];
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ sb.append(new String(buffer, 0, len));
+ }
+ LOG.info("jmx returned: " + sb.toString());
+ assertTrue(sb.toString().contains("JvmMetrics"));
+ return null;
+ }
+ });
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] hadoop git commit: HDFS-12683. DFSZKFailOverController
re-order logic for logging Exception. Contributed by Bharat Viswanadham.
Posted by ae...@apache.org.
HDFS-12683. DFSZKFailOverController re-order logic for logging Exception. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b00c9a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b00c9a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b00c9a4
Branch: refs/heads/HDFS-7240
Commit: 4b00c9a47f94c4d7c673710bbb3b85b5d2807e7b
Parents: 921338c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Oct 23 10:47:16 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Oct 23 10:47:16 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/ha/ZKFailoverController.java | 48 ++++++++++++--------
.../hdfs/tools/DFSZKFailoverController.java | 3 +-
2 files changed, 30 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b00c9a4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index 20a4681..0ebdacd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -184,40 +184,45 @@ public abstract class ZKFailoverController {
}
});
} catch (RuntimeException rte) {
- LOG.error("The failover controller encounters runtime error: " + rte);
throw (Exception)rte.getCause();
}
}
private int doRun(String[] args)
- throws HadoopIllegalArgumentException, IOException, InterruptedException {
+ throws Exception {
try {
initZK();
} catch (KeeperException ke) {
LOG.error("Unable to start failover controller. Unable to connect "
+ "to ZooKeeper quorum at " + zkQuorum + ". Please check the "
+ "configured value for " + ZK_QUORUM_KEY + " and ensure that "
- + "ZooKeeper is running.");
+ + "ZooKeeper is running.", ke);
return ERR_CODE_NO_ZK;
}
- if (args.length > 0) {
- if ("-formatZK".equals(args[0])) {
- boolean force = false;
- boolean interactive = true;
- for (int i = 1; i < args.length; i++) {
- if ("-force".equals(args[i])) {
- force = true;
- } else if ("-nonInteractive".equals(args[i])) {
- interactive = false;
- } else {
- badArg(args[i]);
+ try {
+ if (args.length > 0) {
+ if ("-formatZK".equals(args[0])) {
+ boolean force = false;
+ boolean interactive = true;
+ for (int i = 1; i < args.length; i++) {
+ if ("-force".equals(args[i])) {
+ force = true;
+ } else if ("-nonInteractive".equals(args[i])) {
+ interactive = false;
+ } else {
+ badArg(args[i]);
+ }
}
+ return formatZK(force, interactive);
+ }
+ else {
+ badArg(args[0]);
}
- return formatZK(force, interactive);
- } else {
- badArg(args[0]);
}
+ } catch (Exception e){
+ LOG.error("The failover controller encounters runtime error", e);
+ throw e;
}
if (!elector.parentZNodeExists()) {
@@ -236,11 +241,14 @@ public abstract class ZKFailoverController {
return ERR_CODE_NO_FENCER;
}
- initRPC();
- initHM();
- startRPC();
try {
+ initRPC();
+ initHM();
+ startRPC();
mainLoop();
+ } catch (Exception e) {
+ LOG.error("The failover controller encounters runtime error: ", e);
+ throw e;
} finally {
rpcServer.stopAndJoin();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b00c9a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
index 01cd934..a454d73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java
@@ -194,7 +194,8 @@ public class DFSZKFailoverController extends ZKFailoverController {
try {
retCode = zkfc.run(parser.getRemainingArgs());
} catch (Throwable t) {
- LOG.fatal("Got a fatal error, exiting now", t);
+ LOG.fatal("DFSZKFailOverController exiting due to earlier exception "
+ + t);
}
System.exit(retCode);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2628d1b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2628d1b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2628d1b0
Branch: refs/heads/HDFS-7240
Commit: 2628d1b06aa0b1bec2a5bc08105388ee268f6ad2
Parents: 0fb394b c904d60
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Oct 23 17:36:16 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Oct 23 17:36:16 2017 -0700
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 161 ++---
dev-support/docker/hadoop_env_checks.sh | 15 +-
hadoop-client-modules/hadoop-client/pom.xml | 60 ++
.../org/apache/hadoop/conf/Configuration.java | 106 ++--
.../crypto/key/kms/KMSClientProvider.java | 8 +-
.../fs/CommonConfigurationKeysPublic.java | 9 +
.../org/apache/hadoop/fs/viewfs/ConfigUtil.java | 68 ++-
.../org/apache/hadoop/fs/viewfs/Constants.java | 16 +-
.../org/apache/hadoop/fs/viewfs/InodeTree.java | 358 ++++++++++--
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 13 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 14 +-
.../apache/hadoop/ha/ZKFailoverController.java | 48 +-
.../io/erasurecode/ErasureCodeConstants.java | 1 +
.../io/erasurecode/coder/ErasureCodingStep.java | 5 +-
.../erasurecode/coder/ErasureDecodingStep.java | 5 +-
.../erasurecode/coder/ErasureEncodingStep.java | 5 +-
.../coder/HHXORErasureDecodingStep.java | 12 +-
.../coder/HHXORErasureEncodingStep.java | 10 +-
.../io/erasurecode/coder/util/HHUtil.java | 4 +-
.../rawcoder/AbstractNativeRawDecoder.java | 14 +-
.../rawcoder/AbstractNativeRawEncoder.java | 14 +-
.../rawcoder/NativeRSRawDecoder.java | 11 +-
.../rawcoder/NativeRSRawEncoder.java | 11 +-
.../rawcoder/NativeXORRawDecoder.java | 14 +-
.../rawcoder/NativeXORRawEncoder.java | 9 +-
.../rawcoder/RSLegacyRawDecoder.java | 6 +-
.../erasurecode/rawcoder/RawErasureDecoder.java | 17 +-
.../erasurecode/rawcoder/RawErasureEncoder.java | 16 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 5 +-
.../hadoop/metrics2/source/JvmMetrics.java | 16 +
.../hadoop/util/concurrent/ExecutorHelper.java | 10 +-
.../apache/hadoop/io/erasurecode/jni_common.c | 5 +-
.../hadoop/io/erasurecode/jni_rs_decoder.c | 9 +-
.../hadoop/io/erasurecode/jni_rs_encoder.c | 9 +-
.../hadoop/io/erasurecode/jni_xor_decoder.c | 9 +-
.../hadoop/io/erasurecode/jni_xor_encoder.c | 9 +-
.../src/main/resources/core-default.xml | 21 +
.../src/site/markdown/GroupsMapping.md | 5 +-
.../apache/hadoop/conf/TestConfiguration.java | 15 +
.../hadoop/fs/contract/ContractTestUtils.java | 21 +
.../fs/viewfs/ViewFileSystemBaseTest.java | 4 +-
.../erasurecode/coder/TestErasureCoderBase.java | 18 +-
.../coder/TestHHErasureCoderBase.java | 10 +-
.../rawcoder/RawErasureCoderBenchmark.java | 9 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 15 +-
.../rawcoder/TestNativeRSRawCoder.java | 6 +
.../rawcoder/TestNativeXORRawCoder.java | 7 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 61 +-
.../hadoop/crypto/key/kms/server/KMSACLs.java | 4 +
.../crypto/key/kms/server/KMSConfiguration.java | 15 +
.../crypto/key/kms/server/KMSWebServer.java | 24 +
.../hadoop/crypto/key/kms/server/TestKMS.java | 44 +-
.../hadoop/hdfs/DFSStripedOutputStream.java | 51 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 31 +-
.../hadoop/hdfs/DistributedFileSystem.java | 3 +-
.../apache/hadoop/hdfs/ExceptionLastSeen.java | 75 +++
.../hadoop/hdfs/PositionStripeReader.java | 3 +-
.../hadoop/hdfs/StatefulStripeReader.java | 3 +-
.../org/apache/hadoop/hdfs/StripeReader.java | 7 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 18 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hdfs/qjournal/client/AsyncLogger.java | 2 +-
.../hdfs/qjournal/client/IPCLoggerChannel.java | 43 +-
.../qjournal/client/QuorumJournalManager.java | 35 +-
.../qjournal/protocol/QJournalProtocol.java | 40 +-
.../hdfs/qjournal/protocol/RequestInfo.java | 11 +-
.../QJournalProtocolServerSideTranslatorPB.java | 27 +-
.../QJournalProtocolTranslatorPB.java | 178 ++++--
.../hadoop/hdfs/qjournal/server/Journal.java | 15 +-
.../hdfs/qjournal/server/JournalNode.java | 74 ++-
.../qjournal/server/JournalNodeRpcServer.java | 71 ++-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 24 +-
.../server/blockmanagement/BlockManager.java | 22 +-
.../server/datanode/BlockRecoveryWorker.java | 14 +-
.../StripedBlockChecksumReconstructor.java | 2 +-
.../erasurecode/StripedBlockReconstructor.java | 2 +-
.../namenode/ErasureCodingPolicyManager.java | 20 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 8 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 16 +-
.../hdfs/server/namenode/FSNamesystem.java | 5 +
.../server/namenode/FSPermissionChecker.java | 12 +-
.../hdfs/server/namenode/LeaseManager.java | 57 +-
.../server/namenode/ReencryptionHandler.java | 106 ++--
.../server/namenode/ReencryptionUpdater.java | 29 +-
.../server/namenode/top/metrics/TopMetrics.java | 30 +-
.../server/protocol/InterDatanodeProtocol.java | 7 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 4 +-
.../hdfs/tools/DFSZKFailoverController.java | 3 +-
.../src/main/proto/QJournalProtocol.proto | 12 +
.../src/main/resources/hdfs-default.xml | 2 +-
.../src/site/markdown/HDFSErasureCoding.md | 3 +-
.../hadoop-hdfs/src/site/markdown/ViewFs.md | 44 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 5 +
.../viewfs/TestViewFileSystemLinkFallback.java | 264 +++++++++
.../TestViewFileSystemLinkMergeSlash.java | 234 ++++++++
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 6 +-
.../TestDFSStripedOutputStreamWithFailure.java | 116 +++-
...tputStreamWithFailureWithRandomECPolicy.java | 14 +-
.../apache/hadoop/hdfs/TestDatanodeDeath.java | 3 +-
.../TestDistributedFileSystemWithECFile.java | 37 +-
...dFileSystemWithECFileWithRandomECPolicy.java | 49 ++
.../hadoop/hdfs/TestErasureCodingPolicies.java | 27 +
.../org/apache/hadoop/hdfs/TestFileAppend3.java | 3 +-
.../hadoop/hdfs/TestSnapshotCommands.java | 22 +-
.../TestTrashWithSecureEncryptionZones.java | 1 -
.../qjournal/client/TestEpochsAreUnique.java | 4 +-
.../hdfs/qjournal/client/TestQJMWithFaults.java | 12 +-
.../client/TestQuorumJournalManager.java | 5 +-
.../hdfs/qjournal/server/TestJournal.java | 8 +-
.../hdfs/qjournal/server/TestJournalNode.java | 97 +++
.../qjournal/server/TestJournalNodeSync.java | 7 +
.../blockmanagement/TestBlockManager.java | 111 ++++
.../hdfs/server/federation/MockResolver.java | 5 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 3 +-
.../namenode/TestINodeAttributeProvider.java | 60 +-
.../namenode/TestQuotaWithStripedBlocks.java | 40 +-
...uotaWithStripedBlocksWithRandomECPolicy.java | 50 ++
.../hdfs/server/namenode/TestReencryption.java | 59 +-
.../namenode/ha/TestFailureToReadEdits.java | 25 +-
.../server/namenode/metrics/TestTopMetrics.java | 11 +-
.../src/test/resources/testHDFSConf.xml | 4 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 16 +
.../v2/app/job/impl/TaskAttemptImpl.java | 52 +-
.../mapreduce/v2/util/LocalResourceBuilder.java | 169 ++++++
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 137 +----
.../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 13 +-
.../TestLocalDistributedCacheManager.java | 9 +
.../hadoop/mapreduce/v2/util/TestMRApps.java | 17 +-
.../hadoop-mapreduce-client-core/pom.xml | 6 +
.../org/apache/hadoop/mapred/RecordReader.java | 6 +-
.../java/org/apache/hadoop/mapreduce/Job.java | 226 +++++++
.../hadoop/mapreduce/JobResourceUploader.java | 416 +++++++++++--
.../apache/hadoop/mapreduce/MRJobConfig.java | 71 +++
.../hadoop/mapreduce/SharedCacheConfig.java | 102 ++++
.../src/main/resources/mapred-default.xml | 11 +
.../src/site/markdown/SharedCacheSupport.md | 100 ++++
.../mapreduce/TestJobResourceUploader.java | 76 ++-
.../TestJobResourceUploaderWithSharedCache.java | 365 ++++++++++++
.../org/apache/hadoop/mapred/YARNRunner.java | 54 +-
.../hadoop/mapred/TestLocalJobSubmission.java | 52 ++
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 59 ++
hadoop-project/pom.xml | 1 +
hadoop-project/src/site/site.xml | 1 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 130 ++++-
.../hadoop/fs/azure/security/Constants.java | 1 +
.../TestNativeAzureFileSystemAuthorization.java | 584 ++++++++++++++++---
.../java/org/apache/hadoop/tools/DistCp.java | 10 +-
.../hadoop-sls/src/main/bin/rumen2sls.sh | 3 +
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 3 +
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 1 +
.../sls/scheduler/CapacitySchedulerMetrics.java | 5 -
.../sls/scheduler/FairSchedulerMetrics.java | 17 +-
.../sls/scheduler/FifoSchedulerMetrics.java | 5 +-
.../sls/scheduler/SLSCapacityScheduler.java | 19 -
.../yarn/sls/scheduler/SLSFairScheduler.java | 19 -
.../yarn/sls/scheduler/SchedulerMetrics.java | 66 +--
.../hadoop/yarn/api/records/Resource.java | 14 +
.../yarn/api/records/ResourceInformation.java | 10 +
.../yarn/api/records/ResourceTypeInfo.java | 7 +-
.../api/records/impl/LightWeightResource.java | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 33 ++
.../yarn/util/resource/ResourceUtils.java | 96 +--
.../hadoop/yarn/api/records/TestResource.java | 43 ++
.../distributedshell/TestDistributedShell.java | 1 -
.../yarn/client/api/impl/TestAMRMProxy.java | 4 +
.../api/records/impl/pb/ResourcePBImpl.java | 4 +-
.../tfile/LogAggregationTFileController.java | 9 +-
.../org/apache/hadoop/yarn/util/FSDownload.java | 17 +-
.../hadoop/yarn/util/resource/Resources.java | 11 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 22 +
.../src/main/resources/yarn-default.xml | 42 +-
.../hadoop/yarn/api/TestResourcePBImpl.java | 27 +
.../yarn/util/resource/TestResourceUtils.java | 17 +
.../policies/FederationPolicyUtils.java | 41 +-
.../LocalityMulticastAMRMProxyPolicy.java | 103 +++-
.../router/WeightedRandomRouterPolicy.java | 33 +-
.../BaseContainerTokenSecretManager.java | 8 +-
.../policies/TestFederationPolicyUtils.java | 58 ++
.../TestLocalityMulticastAMRMProxyPolicy.java | 110 +++-
.../server/nodemanager/ContainerExecutor.java | 3 +-
.../hadoop/yarn/server/nodemanager/Context.java | 3 +
.../nodemanager/DefaultContainerExecutor.java | 2 +-
.../nodemanager/LinuxContainerExecutor.java | 10 +-
.../yarn/server/nodemanager/NodeManager.java | 92 ++-
.../nodemanager/NodeStatusUpdaterImpl.java | 38 +-
.../containermanager/ContainerManagerImpl.java | 9 +
.../container/ContainerEventType.java | 4 +-
.../container/ContainerImpl.java | 43 +-
.../launcher/ContainersLauncher.java | 2 -
.../launcher/RecoverPausedContainerLaunch.java | 38 +-
.../launcher/RecoveredContainerLaunch.java | 2 +-
.../linux/privileged/PrivilegedOperation.java | 1 +
.../linux/resources/ResourceHandlerChain.java | 4 +-
.../linux/resources/ResourceHandlerModule.java | 42 +-
.../resources/gpu/GpuResourceAllocator.java | 242 ++++++++
.../resources/gpu/GpuResourceHandlerImpl.java | 153 +++++
.../localizer/ResourceLocalizationService.java | 11 +-
.../NodeResourceUpdaterPlugin.java | 52 ++
.../resourceplugin/ResourcePlugin.java | 83 +++
.../resourceplugin/ResourcePluginManager.java | 106 ++++
.../resourceplugin/gpu/GpuDiscoverer.java | 254 ++++++++
.../gpu/GpuNodeResourceUpdateHandler.java | 66 +++
.../resourceplugin/gpu/GpuResourcePlugin.java | 61 ++
.../scheduler/ContainerScheduler.java | 31 +
.../scheduler/ContainerSchedulerEventType.java | 3 +-
.../recovery/NMLeveldbStateStoreService.java | 62 +-
.../recovery/NMNullStateStoreService.java | 6 +-
.../recovery/NMStateStoreService.java | 10 +-
.../webapp/dao/gpu/GpuDeviceInformation.java | 72 +++
.../dao/gpu/GpuDeviceInformationParser.java | 87 +++
.../webapp/dao/gpu/PerGpuDeviceInformation.java | 165 ++++++
.../webapp/dao/gpu/PerGpuMemoryUsage.java | 58 ++
.../webapp/dao/gpu/PerGpuTemperature.java | 80 +++
.../webapp/dao/gpu/PerGpuUtilizations.java | 50 ++
.../impl/container-executor.c | 41 +-
.../impl/modules/gpu/gpu-module.c | 2 +-
.../container-executor/impl/utils/docker-util.c | 5 +-
.../test/modules/gpu/test-gpu-module.cc | 13 +
.../test/test-container-executor.c | 8 +
.../test/utils/test_docker_util.cc | 103 ++--
.../server/nodemanager/NodeManagerTestBase.java | 164 ++++++
.../TestDefaultContainerExecutor.java | 4 +-
.../nodemanager/TestLinuxContainerExecutor.java | 2 +-
.../TestLinuxContainerExecutorWithMocks.java | 2 +-
.../server/nodemanager/TestNodeManager.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 100 +---
.../amrmproxy/BaseAMRMProxyTest.java | 46 +-
.../TestContainerManagerRecovery.java | 4 +
.../resources/TestResourceHandlerModule.java | 8 +-
.../resources/gpu/TestGpuResourceHandler.java | 382 ++++++++++++
.../TestContainersMonitorResourceChange.java | 2 +-
.../TestResourcePluginManager.java | 261 +++++++++
.../resourceplugin/gpu/TestGpuDiscoverer.java | 123 ++++
.../TestContainerSchedulerQueuing.java | 30 +-
.../recovery/NMMemoryStateStoreService.java | 18 +-
.../TestNMLeveldbStateStoreService.java | 16 +-
.../dao/gpu/TestGpuDeviceInformationParser.java | 50 ++
.../test/resources/nvidia-smi-sample-xml-output | 547 +++++++++++++++++
.../server/resourcemanager/ClientRMService.java | 16 +
.../server/resourcemanager/RMAuditLogger.java | 172 +++++-
.../reservation/FairReservationSystem.java | 13 +
.../resourcemanager/resource/ResourceType.java | 28 -
.../scheduler/fair/FSAppAttempt.java | 19 +
.../scheduler/fair/FSLeafQueue.java | 19 +-
.../scheduler/fair/FSParentQueue.java | 14 -
.../resourcemanager/scheduler/fair/FSQueue.java | 34 ++
.../fair/FairSchedulerConfiguration.java | 8 +-
.../ParameterizedSchedulerTestBase.java | 3 +-
.../server/resourcemanager/TestAppManager.java | 2 -
.../resourcemanager/TestApplicationACLs.java | 90 ++-
.../resourcemanager/TestRMAuditLogger.java | 47 +-
.../resourcemanager/TestSignalContainer.java | 8 +
.../recovery/TestZKRMStateStore.java | 5 +
.../scheduler/fair/TestFairScheduler.java | 39 +-
.../webapp/TestRMWebServicesReservation.java | 28 +-
.../router/webapp/RouterWebServiceUtil.java | 36 +-
.../router/webapp/TestRouterWebServiceUtil.java | 17 +-
.../server/ContainerTokenIdentifierForTest.java | 7 +
.../webproxy/amfilter/AmFilterInitializer.java | 16 +-
.../amfilter/TestAmFilterInitializer.java | 8 +-
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 2 +-
.../hadoop-yarn-ui/src/main/webapp/.bowerrc | 5 +-
263 files changed, 10054 insertions(+), 1786 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2628d1b0/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] hadoop git commit: YARN-7333. container-executor fails to
remove entries from a directory that is not writable or executable.
Contributed by Jason Lowe.
Posted by ae...@apache.org.
YARN-7333. container-executor fails to remove entries from a directory that is not writable or executable. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4540ffd1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4540ffd1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4540ffd1
Branch: refs/heads/HDFS-7240
Commit: 4540ffd15f1259d99a7847c60d44c51f29faa397
Parents: b7ff624
Author: Nathan Roberts <nr...@apache.org>
Authored: Mon Oct 16 16:36:51 2017 -0500
Committer: Nathan Roberts <nr...@apache.org>
Committed: Mon Oct 16 17:00:38 2017 -0500
----------------------------------------------------------------------
.../impl/container-executor.c | 41 +++++++++++++-------
.../test/test-container-executor.c | 8 ++++
2 files changed, 35 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4540ffd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 08d69a5..3b04f88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1731,18 +1731,19 @@ static int unlink_helper(int dirfd, const char *name, int flags) {
}
/**
- * Determine if an entry in a directory is a symlink.
+ * Determine if an entry in a directory is another directory without following
+ * symlinks.
*
* @param dirfd The directory file descriptor, or -1 if there is none.
* @param name If dirfd is -1, this is the path to examine.
* Otherwise, this is the file name in the directory to
* examine.
*
- * @return 0 if the entry is not a symlink
- * 1 if the entry is a symlink
+ * @return 0 if the entry is a symlink or otherwise not a directory
+ * 1 if the entry is a directory
* A negative errno code if we couldn't access the entry.
*/
-static int is_symlink_helper(int dirfd, const char *name)
+static int is_dir_helper(int dirfd, const char *name)
{
struct stat stat;
@@ -1755,7 +1756,7 @@ static int is_symlink_helper(int dirfd, const char *name)
return -errno;
}
}
- return !!S_ISLNK(stat.st_mode);
+ return !!S_ISDIR(stat.st_mode);
}
static int recursive_unlink_helper(int dirfd, const char *name,
@@ -1765,30 +1766,29 @@ static int recursive_unlink_helper(int dirfd, const char *name,
DIR *dfd = NULL;
struct stat stat;
- // Check to see if the file is a symlink. If so, delete the symlink rather
- // than what it points to.
- ret = is_symlink_helper(dirfd, name);
+ // Check to see if the file is a directory. If not then we can unlink it now.
+ ret = is_dir_helper(dirfd, name);
if (ret < 0) {
- // is_symlink_helper failed.
+ // is_dir_helper failed.
if (ret == -ENOENT) {
ret = 0;
goto done;
}
ret = -ret;
- fprintf(LOGFILE, "is_symlink_helper(%s) failed: %s\n",
+ fprintf(LOGFILE, "is_dir_helper(%s) failed: %s\n",
fullpath, strerror(ret));
goto done;
- } else if (ret == 1) {
- // is_symlink_helper determined that the path is a symlink.
+ } else if (ret == 0) {
+ // is_dir_helper determined that the path is not a directory.
ret = unlink_helper(dirfd, name, 0);
if (ret) {
- fprintf(LOGFILE, "failed to unlink symlink %s: %s\n",
+ fprintf(LOGFILE, "failed to unlink %s: %s\n",
fullpath, strerror(ret));
}
goto done;
}
- // Open the file. We use O_NOFOLLOW here to ensure that we if a symlink was
+ // Open the directory. We use O_NOFOLLOW here to ensure that if a symlink was
// swapped in by an attacker, we will fail to follow it rather than deleting
// something we potentially should not.
fd = open_helper(dirfd, name);
@@ -1829,6 +1829,19 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
} else {
+ // make sure the directory has full user permissions
+ // so entries can be deleted
+ if ((stat.st_mode & S_IRWXU) != S_IRWXU) {
+ ret = chmod_helper(dirfd, name, 0700);
+ if (ret) {
+ if (ret == ENOENT) {
+ ret = 0;
+ goto done;
+ }
+ fprintf(LOGFILE, "chmod(%s) failed: %s\n", fullpath, strerror(ret));
+ goto done;
+ }
+ }
dfd = fdopendir(fd);
if (!dfd) {
ret = errno;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4540ffd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 609dd2c..4e8db6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -270,6 +270,8 @@ void test_delete_container() {
char buffer[100000];
sprintf(buffer, "mkdir -p %s/who/let/the/dogs/out/who/who", container_dir);
run(buffer);
+ sprintf(buffer, "mknod %s/who/let/the/dogs/out/who/who/p p", container_dir);
+ run(buffer);
sprintf(buffer, "touch %s", dont_touch);
run(buffer);
@@ -287,9 +289,15 @@ void test_delete_container() {
run(buffer);
sprintf(buffer, "chmod 000 %s/who/let/protect", container_dir);
run(buffer);
+ // create a no execute permission directory
+ sprintf(buffer, "chmod 600 %s/who/let/the", container_dir);
+ run(buffer);
// create a no permission directory
sprintf(buffer, "chmod 000 %s/who/let", container_dir);
run(buffer);
+ // create a no write permission directory
+ sprintf(buffer, "chmod 500 %s/who", container_dir);
+ run(buffer);
// delete container directory
char * dirs[] = {app_dir, 0};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] hadoop git commit: YARN-7311. Fix
TestRMWebServicesReservation parametrization for fair scheduler. (Yufei Gu
via Subru).
Posted by ae...@apache.org.
YARN-7311. Fix TestRMWebServicesReservation parametrization for fair scheduler. (Yufei Gu via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75323394
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75323394
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75323394
Branch: refs/heads/HDFS-7240
Commit: 75323394fbc4211596a2c8fbb5e584f3183f742f
Parents: acabc65
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Oct 17 12:38:06 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Oct 17 12:38:06 2017 -0700
----------------------------------------------------------------------
.../reservation/FairReservationSystem.java | 13 +++++++++
.../webapp/TestRMWebServicesReservation.java | 28 +++++++++++---------
2 files changed, 28 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75323394/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java
index 9bf92c2..611fca8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/FairReservationSystem.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -87,4 +88,16 @@ public class FairReservationSystem extends AbstractReservationSystem {
.getSteadyFairShare();
}
+ @Override
+ public Plan getPlan(String planName) {
+ // make sure plan name is a full queue name in fair scheduler. For example,
+ // "root.default" is the full queue name for "default".
+ FSQueue queue = fairScheduler.getQueueManager().getQueue(planName);
+
+ if (queue != null) {
+ return super.getPlan(queue.getQueueName());
+ } else {
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75323394/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
index 657bec4..02aa65f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
@@ -145,21 +145,9 @@ public class TestRMWebServicesReservation extends JerseyTestBase {
bind(GenericExceptionHandler.class);
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
- Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
- conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
- YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
- CapacitySchedulerConfiguration csconf =
- new CapacitySchedulerConfiguration(conf);
- String[] queues = { "default", "dedicated" };
- csconf.setQueues("root", queues);
- csconf.setCapacity("root.default", 50.0f);
- csconf.setCapacity("root.dedicated", 50.0f);
- csconf.setReservable("root.dedicated", true);
- rm = new MockRM(csconf);
+ rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
if (setAuthFilter) {
filter("/*").through(TestRMCustomAuthFilter.class);
@@ -173,6 +161,16 @@ public class TestRMWebServicesReservation extends JerseyTestBase {
public void configureScheduler() {
conf.set(YarnConfiguration.RM_SCHEDULER,
CapacityScheduler.class.getName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration(conf);
+ String[] queues = { "default", "dedicated" };
+ csconf.setQueues("root", queues);
+ csconf.setCapacity("root.default", 50.0f);
+ csconf.setCapacity("root.dedicated", 50.0f);
+ csconf.setReservable("root.dedicated", true);
+ conf = csconf;
}
}
@@ -189,9 +187,13 @@ public class TestRMWebServicesReservation extends JerseyTestBase {
out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
out.println(" </queue>");
out.println(" <queue name=\"dedicated\">");
+ out.println(" <reservation>");
+ out.println(" </reservation>");
out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
out.println(" </queue>");
out.println("</queue>");
+ out.println("<defaultQueueSchedulingPolicy>drf" +
+ "</defaultQueueSchedulingPolicy>");
out.println("</allocations>");
out.close();
} catch (IOException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] hadoop git commit: HDFS-12448. Make sure user defined erasure
coding policy ID will not overflow. Contributed by Huafeng Wang
Posted by ae...@apache.org.
HDFS-12448. Make sure user defined erasure coding policy ID will not overflow. Contributed by Huafeng Wang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce7cf66e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce7cf66e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce7cf66e
Branch: refs/heads/HDFS-7240
Commit: ce7cf66e5ed74c124afdb5a6902fbf297211cc04
Parents: 0f1c037
Author: Kai Zheng <ka...@intel.com>
Authored: Fri Oct 20 09:42:04 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Fri Oct 20 09:42:04 2017 +0800
----------------------------------------------------------------------
.../io/erasurecode/ErasureCodeConstants.java | 1 +
.../namenode/ErasureCodingPolicyManager.java | 20 +++++++++++----
.../src/site/markdown/HDFSErasureCoding.md | 2 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 27 ++++++++++++++++++++
4 files changed, 44 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index d3c3b6b..73b8f56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -50,6 +50,7 @@ public final class ErasureCodeConstants {
public static final ECSchema REPLICATION_1_2_SCHEMA = new ECSchema(
REPLICATION_CODEC_NAME, 1, 2);
+ public static final byte MAX_POLICY_ID = Byte.MAX_VALUE;
public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
public static final byte REPLICATION_POLICY_ID = (byte) 63;
public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 90699b4..62c7f60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -253,6 +253,14 @@ public final class ErasureCodingPolicyManager {
return p;
}
}
+
+ if (getCurrentMaxPolicyID() == ErasureCodeConstants.MAX_POLICY_ID) {
+ throw new HadoopIllegalArgumentException("Adding erasure coding " +
+ "policy failed because the number of policies stored in the " +
+ "system already reached the threshold, which is " +
+ ErasureCodeConstants.MAX_POLICY_ID);
+ }
+
policy.setName(assignedNewName);
policy.setId(getNextAvailablePolicyID());
this.policiesByName.put(policy.getName(), policy);
@@ -261,12 +269,14 @@ public final class ErasureCodingPolicyManager {
return policy;
}
+ private byte getCurrentMaxPolicyID() {
+ return policiesByID.keySet().stream().max(Byte::compareTo).orElse((byte)0);
+ }
+
private byte getNextAvailablePolicyID() {
- byte currentId = this.policiesByID.keySet().stream()
- .max(Byte::compareTo)
- .filter(id -> id >= ErasureCodeConstants.USER_DEFINED_POLICY_START_ID)
- .orElse(ErasureCodeConstants.USER_DEFINED_POLICY_START_ID);
- return (byte) (currentId + 1);
+ byte nextPolicyID = (byte)(getCurrentMaxPolicyID() + 1);
+ return nextPolicyID > ErasureCodeConstants.USER_DEFINED_POLICY_START_ID ?
+ nextPolicyID : ErasureCodeConstants.USER_DEFINED_POLICY_START_ID;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index a171665..270201a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -193,7 +193,7 @@ Below are the details about each command.
* `[-addPolicies -policyFile <file>]`
- Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB.
+ Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB. Currently HDFS allows the user to add 64 policies in total, and the added policy ID is in range of 64 to 127. Adding policy will fail if there are already 64 policies added.
* `[-listCodecs]`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 2c2b05e..06ac073 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -752,6 +752,33 @@ public class TestErasureCodingPolicies {
}
@Test
+ public void testAddECPoliciesExceeded() throws Exception {
+ ECSchema toAddSchema = new ECSchema("rs", 3, 2);
+ int allowNumPolicies = ErasureCodeConstants.MAX_POLICY_ID -
+ ErasureCodeConstants.USER_DEFINED_POLICY_START_ID + 1;
+ for (int i = 0; i < allowNumPolicies; i++) {
+ ErasureCodingPolicy erasureCodingPolicy = new ErasureCodingPolicy(
+ toAddSchema, 1024 + 1024 * i);
+ ErasureCodingPolicy[] policyArray =
+ new ErasureCodingPolicy[]{erasureCodingPolicy};
+ AddErasureCodingPolicyResponse[] responses =
+ fs.addErasureCodingPolicies(policyArray);
+ assertEquals(1, responses.length);
+ assertTrue(responses[0].isSucceed());
+ assertEquals(responses[0].getPolicy().getId(),
+ ErasureCodeConstants.USER_DEFINED_POLICY_START_ID + i);
+ }
+ ErasureCodingPolicy erasureCodingPolicy = new ErasureCodingPolicy(
+ toAddSchema, 1024 + 1024 * allowNumPolicies);
+ ErasureCodingPolicy[] policyArray =
+ new ErasureCodingPolicy[]{erasureCodingPolicy};
+ AddErasureCodingPolicyResponse[] responses =
+ fs.addErasureCodingPolicies(policyArray);
+ assertEquals(1, responses.length);
+ assertFalse(responses[0].isSucceed());
+ }
+
+ @Test
public void testReplicationPolicy() throws Exception {
ErasureCodingPolicy replicaPolicy =
SystemErasureCodingPolicies.getReplicationPolicy();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] hadoop git commit: YARN-7318. Fix shell check warnings of SLS. (Gergely Novák via wangda)
Posted by ae...@apache.org.
YARN-7318. Fix shell check warnings of SLS. (Gergely Novák via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/281d8360
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/281d8360
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/281d8360
Branch: refs/heads/HDFS-7240
Commit: 281d83604df8341c210cee39bdc745ca793c5afa
Parents: f36cbc8
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Oct 20 15:54:01 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Oct 20 15:54:15 2017 -0700
----------------------------------------------------------------------
hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh | 3 +++
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 3 +++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/281d8360/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
index 565dfe6..cda4e73 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
@@ -60,6 +60,8 @@ function calculate_classpath()
function run_sls_generator()
{
+ local args
+
if [[ -z "${outputprefix}" ]] ; then
outputprefix="sls"
fi
@@ -88,6 +90,7 @@ HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ # shellcheck disable=SC1090
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
else
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/281d8360/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index cbc5bc9..e069a4c 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -81,6 +81,8 @@ function calculate_classpath
function run_simulation() {
+ local args
+
if [[ "${inputsls}" != "" ]] ; then
hadoop_add_param args -inputsls "-inputsls ${inputsls}"
fi
@@ -126,6 +128,7 @@ HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ # shellcheck disable=SC1090
. "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
else
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] hadoop git commit: HDFS-12497. Re-enable
TestDFSStripedOutputStreamWithFailure tests. Contributed by Huafeng Wang.
Posted by ae...@apache.org.
HDFS-12497. Re-enable TestDFSStripedOutputStreamWithFailure tests. Contributed by Huafeng Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0477eff8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0477eff8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0477eff8
Branch: refs/heads/HDFS-7240
Commit: 0477eff8be4505ad2730ec16621105b6df9099ae
Parents: 6b7c87c
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Oct 20 13:27:21 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Oct 20 13:27:21 2017 -0700
----------------------------------------------------------------------
.../TestDFSStripedOutputStreamWithFailure.java | 40 +++++++++++---------
...tputStreamWithFailureWithRandomECPolicy.java | 14 +++----
2 files changed, 29 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0477eff8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index e7fa278..afb4d63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.security.token.Token;
@@ -77,30 +79,29 @@ public class TestDFSStripedOutputStreamWithFailure {
.getLogger().setLevel(Level.ALL);
}
+ private final int cellSize = 64 * 1024; //64k
+ private final int stripesPerBlock = 4;
private ErasureCodingPolicy ecPolicy;
private int dataBlocks;
private int parityBlocks;
- private int cellSize;
- private final int stripesPerBlock = 4;
private int blockSize;
private int blockGroupSize;
private static final int FLUSH_POS =
9 * DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT + 1;
- public ErasureCodingPolicy getEcPolicy() {
- return StripedFileTestUtil.getDefaultECPolicy();
+ public ECSchema getEcSchema() {
+ return StripedFileTestUtil.getDefaultECPolicy().getSchema();
}
/*
* Initialize erasure coding policy.
*/
@Before
- public void init(){
- ecPolicy = getEcPolicy();
+ public void init() {
+ ecPolicy = new ErasureCodingPolicy(getEcSchema(), cellSize);
dataBlocks = ecPolicy.getNumDataUnits();
parityBlocks = ecPolicy.getNumParityUnits();
- cellSize = ecPolicy.getCellSize();
blockSize = cellSize * stripesPerBlock;
blockGroupSize = blockSize * dataBlocks;
dnIndexSuite = getDnIndexSuite();
@@ -189,7 +190,7 @@ public class TestDFSStripedOutputStreamWithFailure {
private List<Integer> lengths;
Integer getLength(int i) {
- return i >= 0 && i < lengths.size()? lengths.get(i): null;
+ return i >= 0 && i < lengths.size() ? lengths.get(i): null;
}
private static final Random RANDOM = new Random();
@@ -220,6 +221,10 @@ public class TestDFSStripedOutputStreamWithFailure {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
dfs = cluster.getFileSystem();
+ AddErasureCodingPolicyResponse[] res =
+ dfs.addErasureCodingPolicies(new ErasureCodingPolicy[]{ecPolicy});
+ ecPolicy = res[0].getPolicy();
+ dfs.enableErasureCodingPolicy(ecPolicy.getName());
DFSTestUtil.enableAllECPolicies(dfs);
dfs.mkdirs(dir);
dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
@@ -241,7 +246,7 @@ public class TestDFSStripedOutputStreamWithFailure {
return conf;
}
- @Test(timeout=240000)
+ @Test(timeout=300000)
public void testMultipleDatanodeFailure56() throws Exception {
runTestWithMultipleFailure(getLength(56));
}
@@ -260,7 +265,8 @@ public class TestDFSStripedOutputStreamWithFailure {
@Test(timeout=240000)
public void testBlockTokenExpired() throws Exception {
- final int length = dataBlocks * (blockSize - cellSize);
+ // Make sure killPos is greater than the length of one stripe
+ final int length = dataBlocks * cellSize * 3;
final HdfsConfiguration conf = newHdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
@@ -300,13 +306,13 @@ public class TestDFSStripedOutputStreamWithFailure {
cluster.triggerHeartbeats();
DatanodeInfo[] info = dfs.getClient().datanodeReport(
DatanodeReportType.LIVE);
- assertEquals("Mismatches number of live Dns ", numDatanodes, info.length);
+ assertEquals("Mismatches number of live Dns", numDatanodes, info.length);
final Path dirFile = new Path(dir, "ecfile");
LambdaTestUtils.intercept(
IOException.class,
"File " + dirFile + " could only be written to " +
numDatanodes + " of the " + dataBlocks + " required nodes for " +
- getEcPolicy().getName(),
+ ecPolicy.getName(),
() -> {
try (FSDataOutputStream out = dfs.create(dirFile, true)) {
out.write("something".getBytes());
@@ -413,7 +419,7 @@ public class TestDFSStripedOutputStreamWithFailure {
cluster.triggerHeartbeats();
DatanodeInfo[] info = dfs.getClient().datanodeReport(
DatanodeReportType.LIVE);
- assertEquals("Mismatches number of live Dns ", numDatanodes, info.length);
+ assertEquals("Mismatches number of live Dns", numDatanodes, info.length);
Path srcPath = new Path(dir, "testAddBlockWhenNoSufficientParityNodes");
int fileLength = cellSize - 1000;
final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
@@ -432,7 +438,7 @@ public class TestDFSStripedOutputStreamWithFailure {
try {
LOG.info("runTest: dn=" + dn + ", length=" + length);
setup(conf);
- runTest(length, new int[]{length/2}, new int[]{dn}, false);
+ runTest(length, new int[]{length / 2}, new int[]{dn}, false);
} catch (Throwable e) {
final String err = "failed, dn=" + dn + ", length=" + length
+ StringUtils.stringifyException(e);
@@ -582,10 +588,10 @@ public class TestDFSStripedOutputStreamWithFailure {
long oldGS = -1; // the old GS before bumping
List<Long> gsList = new ArrayList<>();
final List<DatanodeInfo> killedDN = new ArrayList<>();
- int numKilled=0;
+ int numKilled = 0;
for(; pos.get() < length;) {
final int i = pos.getAndIncrement();
- if (numKilled < killPos.length && i == killPos[numKilled]) {
+ if (numKilled < killPos.length && i == killPos[numKilled]) {
assertTrue(firstGS != -1);
final long gs = getGenerationStamp(stripedOut);
if (numKilled == 0) {
@@ -706,8 +712,6 @@ public class TestDFSStripedOutputStreamWithFailure {
private void run(int offset) {
int base = getBase();
- // TODO: Fix and re-enable these flaky tests. See HDFS-12417.
- assumeTrue("Test has been temporarily disabled. See HDFS-12417.", false);
assumeTrue(base >= 0);
final int i = offset + base;
final Integer length = getLength(i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0477eff8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java
index 7fc5233..f671149 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* This tests write operation of DFS striped file with a random erasure code
@@ -28,18 +28,18 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
public class TestDFSStripedOutputStreamWithFailureWithRandomECPolicy extends
TestDFSStripedOutputStreamWithFailure {
+ private final ECSchema schema;
+
private static final Log LOG = LogFactory.getLog(
TestDFSStripedOutputStreamWithRandomECPolicy.class.getName());
- private ErasureCodingPolicy ecPolicy;
-
public TestDFSStripedOutputStreamWithFailureWithRandomECPolicy() {
- ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
- LOG.info(ecPolicy);
+ schema = StripedFileTestUtil.getRandomNonDefaultECPolicy().getSchema();
+ LOG.info(schema);
}
@Override
- public ErasureCodingPolicy getEcPolicy() {
- return ecPolicy;
+ public ECSchema getEcSchema() {
+ return schema;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] hadoop git commit: YARN-7372.
TestContainerSchedulerQueuing.testContainerUpdateExecTypeGuaranteedToOpportunistic
is flaky.
Posted by ae...@apache.org.
YARN-7372. TestContainerSchedulerQueuing.testContainerUpdateExecTypeGuaranteedToOpportunistic is flaky.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/480187ae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/480187ae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/480187ae
Branch: refs/heads/HDFS-7240
Commit: 480187aebbc13547af06684820a416d22e7c4649
Parents: 0477eff
Author: Haibo Chen <ha...@apache.org>
Authored: Fri Oct 20 14:24:17 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Oct 20 14:24:17 2017 -0700
----------------------------------------------------------------------
.../TestContainerSchedulerQueuing.java | 30 ++++++++++++++++----
1 file changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480187ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index ed4f290..5c72e7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -21,14 +21,17 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import com.google.common.base.Supplier;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
@@ -1247,7 +1250,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
@Test
public void testContainerUpdateExecTypeGuaranteedToOpportunistic()
- throws IOException, YarnException, InterruptedException {
+ throws Exception {
delayContainers = true;
containerManager.start();
// Construct the Container-id
@@ -1287,11 +1290,28 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
1, updateResponse.getSuccessfullyUpdatedContainers().size());
Assert.assertTrue(updateResponse.getFailedRequests().isEmpty());
- //Make sure the container is running
- List<ContainerId> statList = new ArrayList<ContainerId>();
- statList.add(cId);
GetContainerStatusesRequest statRequest =
- GetContainerStatusesRequest.newInstance(statList);
+ GetContainerStatusesRequest.newInstance(Collections.singletonList(cId));
+ GenericTestUtils.waitFor(
+ new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ Assert.assertEquals(1, containerStatuses.size());
+
+ ContainerStatus status = containerStatuses.get(0);
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+ status.getState());
+
+ return status.getExecutionType() == ExecutionType.OPPORTUNISTIC;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }, 100, 10000);
List<ContainerStatus> containerStatuses = containerManager
.getContainerStatuses(statRequest).getContainerStatuses();
Assert.assertEquals(1, containerStatuses.size());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] hadoop git commit: HDFS-12620. Backporting HDFS-10467 to
branch-2. Contributed by Inigo Goiri.
Posted by ae...@apache.org.
HDFS-12620. Backporting HDFS-10467 to branch-2. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f1c0376
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f1c0376
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f1c0376
Branch: refs/heads/HDFS-7240
Commit: 0f1c0376186de6446d595be7bb445ed6b71ae499
Parents: 7b4b018
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Oct 19 18:08:45 2017 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Oct 19 18:08:45 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/federation/MockResolver.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f1c0376/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index a481553..151d731 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -264,7 +264,10 @@ public class MockResolver
@Override
public PathLocation getDestinationForPath(String path) throws IOException {
List<RemoteLocation> remoteLocations = new LinkedList<>();
- for (String key : this.locations.keySet()) {
+ // We go from the leaves to the root
+ List<String> keys = new ArrayList<>(this.locations.keySet());
+ Collections.sort(keys, Collections.reverseOrder());
+ for (String key : keys) {
if (path.startsWith(key)) {
for (RemoteLocation location : this.locations.get(key)) {
String finalPath = location.getDest() + path.substring(key.length());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] hadoop git commit: HADOOP-14948. Document missing config key
hadoop.treat.subject.external. Contributed by Ajay Kumar.
Posted by ae...@apache.org.
HADOOP-14948. Document missing config key hadoop.treat.subject.external. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e906108f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e906108f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e906108f
Branch: refs/heads/HDFS-7240
Commit: e906108fc98a011630d12a43e557b81d7ef7ea5d
Parents: 8a61525
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Oct 16 16:42:59 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Oct 16 16:42:59 2017 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/resources/core-default.xml | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e906108f/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e7ea10f..bde7a85 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2763,4 +2763,17 @@
the ZK CLI).
</description>
</property>
+ <property>
+ <name>hadoop.treat.subject.external</name>
+ <value>false</value>
+ <description>
+ When creating UGI with UserGroupInformation(Subject), treat the passed
+ subject external if set to true, and assume the owner of the subject
+ should do the credential renewal.
+
+ When true this property will introduce an incompatible change which
+ may require changes in client code. For more details, see the jiras:
+ HADOOP-13805,HADOOP-13558.
+ </description>
+ </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] hadoop git commit: YARN-4090. Make Collections.sort() more
efficient by caching resource usage. (Contributed by Yufei Gu,
Shilong Zhang and Xianyin Xin)
Posted by ae...@apache.org.
YARN-4090. Make Collections.sort() more efficient by caching resource usage. (Contributed by Yufei Gu, Shilong Zhang and Xianyin Xin)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f4cdf10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f4cdf10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f4cdf10
Branch: refs/heads/HDFS-7240
Commit: 1f4cdf10681b6903207a63fb5c306c9665ed9464
Parents: 4afd308
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Oct 20 01:32:20 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Oct 20 01:32:20 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/FSAppAttempt.java | 19 ++++++++++
.../scheduler/fair/FSLeafQueue.java | 19 +---------
.../scheduler/fair/FSParentQueue.java | 14 -------
.../resourcemanager/scheduler/fair/FSQueue.java | 34 +++++++++++++++++
.../scheduler/fair/TestFairScheduler.java | 39 +++++++++++++++++++-
5 files changed, 93 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3024558..8ab6e13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
@@ -167,6 +168,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
this.attemptResourceUsage.decUsed(containerResource);
+ getQueue().decUsedResource(containerResource);
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
@@ -461,6 +463,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
type, node, schedulerKey, container);
this.attemptResourceUsage.incUsed(container.getResource());
+ getQueue().incUsedResource(container.getResource());
// Update resource requests related to "request" and store in RMContainer
((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
@@ -644,6 +647,22 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
schedulerKey.getAllocationRequestId());
}
+ @Override
+ public synchronized void recoverContainer(SchedulerNode node,
+ RMContainer rmContainer) {
+ try {
+ writeLock.lock();
+
+ super.recoverContainer(node, rmContainer);
+
+ if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) {
+ getQueue().incUsedResource(rmContainer.getContainer().getResource());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
/**
* Reserve a spot for {@code container} on this {@code node}. If
* the container is {@code alreadyReserved} on the node, simply
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 60477d4..49d2166 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -89,6 +89,7 @@ public class FSLeafQueue extends FSQueue {
} else {
nonRunnableApps.add(app);
}
+ incUsedResource(app.getResourceUsage());
} finally {
writeLock.unlock();
}
@@ -123,6 +124,7 @@ public class FSLeafQueue extends FSQueue {
getMetrics().setAMResourceUsage(amResourceUsage);
}
+ decUsedResource(app.getResourceUsage());
return runnable;
}
@@ -292,23 +294,6 @@ public class FSLeafQueue extends FSQueue {
return demand;
}
- @Override
- public Resource getResourceUsage() {
- Resource usage = Resources.createResource(0);
- readLock.lock();
- try {
- for (FSAppAttempt app : runnableApps) {
- Resources.addTo(usage, app.getResourceUsage());
- }
- for (FSAppAttempt app : nonRunnableApps) {
- Resources.addTo(usage, app.getResourceUsage());
- }
- } finally {
- readLock.unlock();
- }
- return usage;
- }
-
Resource getAmResourceUsage() {
return amResourceUsage;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index e42c9f5..a8e53fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -119,20 +119,6 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public Resource getResourceUsage() {
- Resource usage = Resources.createResource(0);
- readLock.lock();
- try {
- for (FSQueue child : childQueues) {
- Resources.addTo(usage, child.getResourceUsage());
- }
- } finally {
- readLock.unlock();
- }
- return usage;
- }
-
- @Override
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index e0df480..4babfd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -57,6 +57,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private Resource fairShare = Resources.createResource(0, 0);
private Resource steadyFairShare = Resources.createResource(0, 0);
private Resource reservedResource = Resources.createResource(0, 0);
+ private final Resource resourceUsage = Resource.newInstance(0, 0);
private final String name;
protected final FairScheduler scheduler;
private final YarnAuthorizationProvider authorizer;
@@ -479,6 +480,39 @@ public abstract class FSQueue implements Queue, Schedulable {
}
@Override
+ public Resource getResourceUsage() {
+ return resourceUsage;
+ }
+
+ /**
+ * Increase resource usage for this queue and all parent queues.
+ *
+ * @param res the resource to increase
+ */
+ protected void incUsedResource(Resource res) {
+ synchronized (resourceUsage) {
+ Resources.addTo(resourceUsage, res);
+ if (parent != null) {
+ parent.incUsedResource(res);
+ }
+ }
+ }
+
+ /**
+ * Decrease resource usage for this queue and all parent queues.
+ *
+ * @param res the resource to decrease
+ */
+ protected void decUsedResource(Resource res) {
+ synchronized (resourceUsage) {
+ Resources.subtractFrom(resourceUsage, res);
+ if (parent != null) {
+ parent.decUsedResource(res);
+ }
+ }
+ }
+
+ @Override
public Priority getDefaultApplicationPriority() {
// TODO add implementation for FSParentQueue
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 6b2109d..944db1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4585,6 +4585,43 @@ public class TestFairScheduler extends FairSchedulerTestBase {
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
scheduler.getSchedulerApplications(), scheduler, "default");
}
+
+ @Test
+ public void testResourceUsageByMoveApp() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node1 = MockNodes.newNodeInfo(
+ 1, Resources.createResource(1 * GB, 4), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1 * GB, 2, "parent1.queue1", "user1", 2);
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(updateEvent);
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSQueue parent1 = queueMgr.getParentQueue("parent1", true);
+ FSQueue parent2 = queueMgr.getParentQueue("parent2", true);
+ FSQueue queue2 = queueMgr.getLeafQueue("parent2.queue2", true);
+ FSQueue queue1 = queueMgr.getLeafQueue("parent1.queue1", true);
+
+ Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 1 * GB);
+
+ scheduler.moveApplication(appAttId.getApplicationId(), "parent2.queue2");
+
+ Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 0);
+ }
@Test (expected = YarnException.class)
public void testMoveWouldViolateMaxAppsConstraints() throws Exception {
@@ -4598,7 +4635,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
-
+
scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] hadoop git commit: HADOOP-14771. hadoop-client does not
include hadoop-yarn-client. (Ajay Kumar via Haibo Chen)
Posted by ae...@apache.org.
HADOOP-14771. hadoop-client does not include hadoop-yarn-client. (Ajay Kumar via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3dd3d1dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3dd3d1dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3dd3d1dd
Branch: refs/heads/HDFS-7240
Commit: 3dd3d1dd77515be6d62e637c27d7d37d24058617
Parents: 4ab0c8f
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Oct 19 13:25:08 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Oct 19 13:25:08 2017 -0700
----------------------------------------------------------------------
hadoop-client-modules/hadoop-client/pom.xml | 60 ++++++++++++++++++++++++
1 file changed, 60 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3dd3d1dd/hadoop-client-modules/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
index bed3f5c..a738d47 100644
--- a/hadoop-client-modules/hadoop-client/pom.xml
+++ b/hadoop-client-modules/hadoop-client/pom.xml
@@ -179,6 +179,66 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <!--Excluding hadoop-yarn-api & hadoop-annotations as they are already
+ included as direct dependencies. Guava,commons-cli and log4j are
+ transitive dependencies -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-servlet</artifactId>
+ </exclusion>
+
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>compile</scope>
<exclusions>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] hadoop git commit: HADOOP-14966. Handle JDK-8071638 for
hadoop-common. Contributed by Bibin A Chundatt.
Posted by ae...@apache.org.
HADOOP-14966. Handle JDK-8071638 for hadoop-common. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b812245
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b812245
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b812245
Branch: refs/heads/HDFS-7240
Commit: 6b8122458ede9ececb138bed99d368580e5439a1
Parents: 4b00c9a
Author: Naganarasimha <na...@apache.org>
Authored: Mon Oct 23 23:41:38 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Oct 23 23:42:06 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/util/concurrent/ExecutorHelper.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b812245/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java
index 02452a3..0b34951 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java
@@ -40,14 +40,16 @@ public final class ExecutorHelper {
//For additional information, see: https://docs.oracle
// .com/javase/7/docs/api/java/util/concurrent/ThreadPoolExecutor
- // .html#afterExecute(java.lang.Runnable,%20java.lang.Throwable) .
+ // .html#afterExecute(java.lang.Runnable,%20java.lang.Throwable)
- if (t == null && r instanceof Future<?>) {
+ // Handle JDK-8071638
+ if (t == null && r instanceof Future<?> && ((Future<?>) r).isDone()) {
try {
((Future<?>) r).get();
} catch (ExecutionException ee) {
- LOG.warn("Execution exception when running task in " +
- Thread.currentThread().getName());
+ LOG.warn(
+ "Execution exception when running task in " + Thread.currentThread()
+ .getName());
t = ee.getCause();
} catch (InterruptedException ie) {
LOG.warn("Thread (" + Thread.currentThread() + ") interrupted: ", ie);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] hadoop git commit: HDFS-12650. Use slf4j instead of log4j in
LeaseManager. Contributed by Ajay Kumar.
Posted by ae...@apache.org.
HDFS-12650. Use slf4j instead of log4j in LeaseManager. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96be7956
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96be7956
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96be7956
Branch: refs/heads/HDFS-7240
Commit: 96be795656039c0c12031ec31becfcd35fbb1b70
Parents: 6b81224
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Oct 23 13:20:46 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Oct 23 13:20:46 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/LeaseManager.java | 57 +++++++++-----------
.../server/protocol/InterDatanodeProtocol.java | 7 ++-
.../apache/hadoop/hdfs/TestDatanodeDeath.java | 3 +-
.../org/apache/hadoop/hdfs/TestFileAppend3.java | 3 +-
4 files changed, 33 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96be7956/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 45699cb..3746e13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -37,8 +37,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -50,6 +48,8 @@ import org.apache.hadoop.util.Daemon;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* LeaseManager does the lease housekeeping for writing on files.
@@ -75,7 +75,8 @@ import org.apache.hadoop.util.Time;
*/
@InterfaceAudience.Private
public class LeaseManager {
- public static final Log LOG = LogFactory.getLog(LeaseManager.class);
+ public static final Logger LOG = LoggerFactory.getLogger(LeaseManager.class
+ .getName());
private final FSNamesystem fsnamesystem;
private long softLimit = HdfsConstants.LEASE_SOFTLIMIT_PERIOD;
private long hardLimit = HdfsConstants.LEASE_HARDLIMIT_PERIOD;
@@ -142,8 +143,8 @@ public class LeaseManager {
for (Long id : getINodeIdWithLeases()) {
final INodeFile cons = fsnamesystem.getFSDirectory().getInode(id).asFile();
if (!cons.isUnderConstruction()) {
- LOG.warn("The file " + cons.getFullPathName()
- + " is not under construction but has lease.");
+ LOG.warn("The file {} is not under construction but has lease.",
+ cons.getFullPathName());
continue;
}
BlockInfo[] blocks = cons.getBlocks();
@@ -155,7 +156,7 @@ public class LeaseManager {
numUCBlocks++;
}
}
- LOG.info("Number of blocks under construction: " + numUCBlocks);
+ LOG.info("Number of blocks under construction: {}", numUCBlocks);
return numUCBlocks;
}
@@ -250,9 +251,8 @@ public class LeaseManager {
}
final long endTimeMs = Time.monotonicNow();
if ((endTimeMs - startTimeMs) > 1000) {
- LOG.info("Took " + (endTimeMs - startTimeMs) + " ms to collect "
- + iipSet.size() + " open files with leases" +
- ((ancestorDir != null) ?
+ LOG.info("Took {} ms to collect {} open files with leases {}",
+ (endTimeMs - startTimeMs), iipSet.size(), ((ancestorDir != null) ?
" under " + ancestorDir.getFullPathName() : "."));
}
return iipSet;
@@ -287,8 +287,8 @@ public class LeaseManager {
final INodeFile inodeFile =
fsnamesystem.getFSDirectory().getInode(inodeId).asFile();
if (!inodeFile.isUnderConstruction()) {
- LOG.warn("The file " + inodeFile.getFullPathName()
- + " is not under construction but has lease.");
+ LOG.warn("The file {} is not under construction but has lease.",
+ inodeFile.getFullPathName());
continue;
}
openFileEntries.add(new OpenFileEntry(
@@ -348,16 +348,13 @@ public class LeaseManager {
private synchronized void removeLease(Lease lease, long inodeId) {
leasesById.remove(inodeId);
if (!lease.removeFile(inodeId)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("inode " + inodeId + " not found in lease.files (=" + lease
- + ")");
- }
+ LOG.debug("inode {} not found in lease.files (={})", inodeId, lease);
}
if (!lease.hasFiles()) {
leases.remove(lease.holder);
if (!sortedLeases.remove(lease)) {
- LOG.error(lease + " not found in sortedLeases");
+ LOG.error("{} not found in sortedLeases", lease);
}
}
}
@@ -370,8 +367,8 @@ public class LeaseManager {
if (lease != null) {
removeLease(lease, src.getId());
} else {
- LOG.warn("Removing non-existent lease! holder=" + holder +
- " src=" + src.getFullPathName());
+ LOG.warn("Removing non-existent lease! holder={} src={}", holder, src
+ .getFullPathName());
}
}
@@ -513,9 +510,7 @@ public class LeaseManager {
Thread.sleep(fsnamesystem.getLeaseRecheckIntervalMs());
} catch(InterruptedException ie) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(name + " is interrupted", ie);
- }
+ LOG.debug("{} is interrupted", name, ie);
} catch(Throwable e) {
LOG.warn("Unexpected throwable: ", e);
}
@@ -537,7 +532,7 @@ public class LeaseManager {
sortedLeases.first().expiredHardLimit()
&& !isMaxLockHoldToReleaseLease(start)) {
Lease leaseToCheck = sortedLeases.first();
- LOG.info(leaseToCheck + " has expired hard limit");
+ LOG.info("{} has expired hard limit", leaseToCheck);
final List<Long> removing = new ArrayList<>();
// need to create a copy of the oldest lease files, because
@@ -568,16 +563,16 @@ public class LeaseManager {
completed = fsnamesystem.internalReleaseLease(
leaseToCheck, p, iip, newHolder);
} catch (IOException e) {
- LOG.warn("Cannot release the path " + p + " in the lease "
- + leaseToCheck + ". It will be retried.", e);
+ LOG.warn("Cannot release the path {} in the lease {}. It will be "
+ + "retried.", p, leaseToCheck, e);
continue;
}
if (LOG.isDebugEnabled()) {
if (completed) {
- LOG.debug("Lease recovery for inode " + id + " is complete. " +
- "File closed.");
+ LOG.debug("Lease recovery for inode {} is complete. File closed"
+ + ".", id);
} else {
- LOG.debug("Started block recovery " + p + " lease " + leaseToCheck);
+ LOG.debug("Started block recovery {} lease {}", p, leaseToCheck);
}
}
// If a lease recovery happened, we need to sync later.
@@ -585,13 +580,13 @@ public class LeaseManager {
needSync = true;
}
} catch (IOException e) {
- LOG.warn("Removing lease with an invalid path: " + p + ","
- + leaseToCheck, e);
+ LOG.warn("Removing lease with an invalid path: {},{}", p,
+ leaseToCheck, e);
removing.add(id);
}
if (isMaxLockHoldToReleaseLease(start)) {
- LOG.debug("Breaking out of checkLeases after " +
- fsnamesystem.getMaxLockHoldToReleaseLeaseMs() + "ms.");
+ LOG.debug("Breaking out of checkLeases after {} ms.",
+ fsnamesystem.getMaxLockHoldToReleaseLeaseMs());
break;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96be7956/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
index 72cb0c1..19cb36a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
@@ -19,14 +19,13 @@
package org.apache.hadoop.hdfs.server.protocol;
import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.security.KerberosInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** An inter-datanode protocol for updating generation stamp
*/
@@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface InterDatanodeProtocol {
- public static final Log LOG = LogFactory.getLog(InterDatanodeProtocol.class);
+ Logger LOG = LoggerFactory.getLogger(InterDatanodeProtocol.class.getName());
/**
* Until version 9, this class InterDatanodeProtocol served as both
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96be7956/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
index d75e9ae..a1aae92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
@@ -45,7 +45,8 @@ public class TestDatanodeDeath {
DFSTestUtil.setNameNodeLogLevel(Level.ALL);
GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, org.slf4j
+ .event.Level.TRACE);
}
static final int blockSize = 8192;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96be7956/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
index 440666f..6cb63a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
@@ -58,7 +58,8 @@ public class TestFileAppend3 {
DFSTestUtil.setNameNodeLogLevel(Level.ALL);
GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, org.slf4j
+ .event.Level.TRACE);
}
static final long BLOCK_SIZE = 64 * 1024;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] hadoop git commit: HADOOP-14954. MetricsSystemImpl#init
should increment refCount when already initialized. Contributed by John
Zhuge.
Posted by ae...@apache.org.
HADOOP-14954. MetricsSystemImpl#init should increment refCount when already initialized. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14b3c269
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14b3c269
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14b3c269
Branch: refs/heads/HDFS-7240
Commit: 14b3c2695b152285ed2919f2aa5a25e247c47871
Parents: 281d836
Author: John Zhuge <jz...@cloudera.com>
Authored: Mon Oct 16 13:29:29 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Oct 20 16:13:41 2017 -0700
----------------------------------------------------------------------
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 12 ++++-
.../metrics2/impl/TestMetricsSystemImpl.java | 46 ++++++++++++++++++++
2 files changed, 57 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b3c269/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index ee1672e..c507a77 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -140,6 +140,16 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
this(null);
}
+ @VisibleForTesting
+ boolean isMonitoring() {
+ return monitoring;
+ }
+
+ @VisibleForTesting
+ int getRefCount() {
+ return refCount;
+ }
+
/**
* Initialized the metrics system with a prefix.
* @param prefix the system will look for configs with the prefix
@@ -147,12 +157,12 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
*/
@Override
public synchronized MetricsSystem init(String prefix) {
+ ++refCount;
if (monitoring && !DefaultMetricsSystem.inMiniClusterMode()) {
LOG.warn(this.prefix +" metrics system already initialized!");
return this;
}
this.prefix = checkNotNull(prefix, "prefix");
- ++refCount;
if (monitoring) {
// in mini cluster mode
LOG.info(this.prefix +" metrics system started (again)");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b3c269/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
index abd1b13..879de2d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
@@ -26,6 +26,8 @@ import java.util.concurrent.atomic.*;
import javax.annotation.Nullable;
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -559,6 +561,50 @@ public class TestMetricsSystemImpl {
ms.shutdown();
}
+ @Test
+ public void testInitShutdown() {
+ boolean oldMode = DefaultMetricsSystem.inMiniClusterMode();
+ try {
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ runInitShutdownTests();
+
+ DefaultMetricsSystem.setMiniClusterMode(false);
+ runInitShutdownTests();
+ } finally {
+ DefaultMetricsSystem.setMiniClusterMode(oldMode);
+ }
+ }
+
+ private void runInitShutdownTests() {
+ MetricsSystemImpl ms = new MetricsSystemImpl();
+ Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(false));
+ Assert.assertThat(ms.getRefCount(), CoreMatchers.is(0));
+
+ ms.init("TestMetricsSystem1");
+ Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
+ Assert.assertThat(ms.getRefCount(), CoreMatchers.is(1));
+
+ ms.shutdown();
+ Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(false));
+ Assert.assertThat(ms.getRefCount(), CoreMatchers.is(0));
+
+ ms.init("TestMetricsSystem2");
+ Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
+ Assert.assertThat(ms.getRefCount(), CoreMatchers.is(1));
+
+ ms.init("TestMetricsSystem3");
+ Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
+ Assert.assertThat(ms.getRefCount(), CoreMatchers.is(2));
+
+ ms.shutdown();
+ Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
+ Assert.assertThat(ms.getRefCount(), CoreMatchers.is(1));
+
+ ms.shutdown();
+ Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(false));
+ Assert.assertThat(ms.getRefCount(), CoreMatchers.is(0));
+ }
+
@Metrics(context="test")
private static class TestSource {
@Metric("C1 desc") MutableCounterLong c1;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] hadoop git commit: HADOOP-14816. Update Dockerfile to use
Xenial. Contributed by Allen Wittenauer
Posted by ae...@apache.org.
HADOOP-14816. Update Dockerfile to use Xenial. Contributed by Allen Wittenauer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca8ddc6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca8ddc6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca8ddc6a
Branch: refs/heads/HDFS-7240
Commit: ca8ddc6aa413de347866ad9a0a3407356a280a1f
Parents: cbd2b73
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Oct 19 16:33:47 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Oct 19 16:45:18 2017 -0700
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 161 +++++++++++++++------------
dev-support/docker/hadoop_env_checks.sh | 15 ++-
2 files changed, 98 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8ddc6a/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 31ac611..8af002d 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -18,21 +18,28 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.
-
-FROM ubuntu:trusty
+FROM ubuntu:xenial
WORKDIR /root
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
+
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
######
-# Install common dependencies from packages
+# Install common dependencies from packages. Versions here are either
+# sufficient or irrelevant.
#
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
# Ubuntu Java. See Java section below!
######
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+RUN apt-get -q update && apt-get -q install -y \
+ apt-utils \
build-essential \
bzip2 \
curl \
@@ -42,7 +49,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
gcc \
git \
gnupg-agent \
- make \
libbz2-dev \
libcurl4-openssl-dev \
libfuse-dev \
@@ -51,106 +57,110 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
libsnappy-dev \
libssl-dev \
libtool \
+ locales \
+ make \
pinentry-curses \
pkg-config \
- protobuf-compiler \
- protobuf-c-compiler \
python \
python2.7 \
- python2.7-dev \
python-pip \
+ python-pkg-resources \
+ python-setuptools \
+ python-wheel \
rsync \
+ software-properties-common \
snappy \
+ sudo \
zlib1g-dev
#######
-# Oracle Java
+# OpenJDK 8
#######
+RUN apt-get -q install -y openjdk-8-jdk
-RUN echo "dot_style = mega" > "/root/.wgetrc"
-RUN echo "quiet = on" >> "/root/.wgetrc"
-
-RUN apt-get -q install --no-install-recommends -y software-properties-common
-RUN add-apt-repository -y ppa:webupd8team/java
-RUN apt-get -q update
-
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get -q install --no-install-recommends -y oracle-java8-installer
-
-####
-# Apps that require Java
-###
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant
+#######
+# OpenJDK 9
+# w/workaround for
+# https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191
+#######
+RUN apt-get -o Dpkg::Options::="--force-overwrite" \
+ -q install -y \
+ openjdk-9-jdk-headless
-######
-# Install Apache Maven
-######
-RUN mkdir -p /opt/maven && \
- curl -L -s -S \
- https://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
- -o /opt/maven.tar.gz && \
- tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven
-ENV MAVEN_HOME /opt/maven
-ENV PATH "${PATH}:/opt/maven/bin"
+#######
+# Set default Java
+#######
+#
+# By default, OpenJDK sets the default Java to the highest version.
+# We want the opposite, soooooo....
+#
+RUN update-java-alternatives --set java-1.8.0-openjdk-amd64
+RUN update-alternatives --get-selections | grep -i jdk | \
+ while read line; do \
+ alternative=$(echo $line | awk '{print $1}'); \
+ path=$(echo $line | awk '{print $3}'); \
+ newpath=$(echo $path | sed -e 's/java-9/java-8/'); \
+ update-alternatives --set $alternative $newpath; \
+ done
######
-# Install cmake
+# Install cmake 3.1.0 (3.5.1 ships with Xenial)
######
RUN mkdir -p /opt/cmake && \
curl -L -s -S \
- https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
- -o /opt/cmake.tar.gz && \
+ https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
+ -o /opt/cmake.tar.gz && \
tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
ENV CMAKE_HOME /opt/cmake
ENV PATH "${PATH}:/opt/cmake/bin"
######
-# Install findbugs
+# Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial)
######
-RUN mkdir -p /opt/findbugs && \
+RUN mkdir -p /opt/protobuf-src && \
curl -L -s -S \
- https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
- -o /opt/findbugs.tar.gz && \
- tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
-ENV FINDBUGS_HOME /opt/findbugs
-ENV PATH "${PATH}:/opt/findbugs/bin"
+ https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
+ -o /opt/protobuf.tar.gz && \
+ tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src
+RUN cd /opt/protobuf-src && ./configure --prefix=/opt/protobuf && make install
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+######
+# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
+######
+RUN apt-get -q update && apt-get -q install -y maven
+ENV MAVEN_HOME /usr
+
+######
+# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
+######
+RUN apt-get -q update && apt-get -q install -y findbugs
+ENV FINDBUGS_HOME /usr
####
-# Install shellcheck
+# Install shellcheck (0.4.6, the latest as of 2017-09-26)
####
-RUN apt-get -q install -y cabal-install
-RUN mkdir /root/.cabal
-RUN echo "remote-repo: hackage.fpcomplete.com:http://hackage.fpcomplete.com/" >> /root/.cabal/config
-#RUN echo "remote-repo: hackage.haskell.org:http://hackage.haskell.org/" > /root/.cabal/config
-RUN echo "remote-repo-cache: /root/.cabal/packages" >> /root/.cabal/config
-RUN cabal update
-RUN cabal install shellcheck --global
+RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2
+RUN apt-get -q update && apt-get -q install -y shellcheck
####
-# Install bats
+# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
####
-RUN add-apt-repository -y ppa:duggan/bats
-RUN apt-get -q update
-RUN apt-get -q install --no-install-recommends -y bats
+RUN apt-get -q update && apt-get -q install -y bats
####
-# Install pylint
+# Install pylint (always want latest)
####
-RUN pip install pylint
+RUN pip2 install pylint
####
# Install dateutil.parser
####
-RUN pip install python-dateutil
+RUN pip2 install python-dateutil
###
-# Avoid out of memory errors in builds
-###
-ENV MAVEN_OPTS -Xms256m -Xmx512m
-
-###
-# Install node js tools for web UI frameowkr
+# Install node.js for web UI framework (4.2.6 ships with Xenial)
###
RUN apt-get -y install nodejs && \
ln -s /usr/bin/nodejs /usr/bin/node && \
@@ -160,18 +170,29 @@ RUN apt-get -y install nodejs && \
npm install -g ember-cli
###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx1g
+
+
+###
# Everything past this point is either not needed for testing or breaks Yetus.
# So tell Yetus not to read the rest of the file:
# YETUS CUT HERE
###
####
-# Install Forrest (for Apache Hadoop website)
+# Install svn, Ant, & Forrest (for Apache Hadoop website)
###
-RUN mkdir -p /usr/local/apache-forrest ; \
- curl -s -S -O http://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz ; \
- tar xzf *forrest* --strip-components 1 -C /usr/local/apache-forrest ; \
- echo 'forrest.home=/usr/local/apache-forrest' > build.properties
+RUN apt-get -q update && apt-get -q install -y ant subversion
+
+RUN mkdir -p /opt/apache-forrest && \
+ curl -L -s -S \
+ https://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz \
+ -o /opt/forrest.tar.gz && \
+ tar xzf /opt/forrest.tar.gz --strip-components 1 -C /opt/apache-forrest
+RUN echo 'forrest.home=/opt/apache-forrest' > build.properties
+ENV FORREST_HOME=/opt/apache-forrest
# Add a welcome message and environment checks.
ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8ddc6a/dev-support/docker/hadoop_env_checks.sh
----------------------------------------------------------------------
diff --git a/dev-support/docker/hadoop_env_checks.sh b/dev-support/docker/hadoop_env_checks.sh
index 5cb4b2b..c9420e7 100755
--- a/dev-support/docker/hadoop_env_checks.sh
+++ b/dev-support/docker/hadoop_env_checks.sh
@@ -16,6 +16,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# SHELLDOC-IGNORE
+
# -------------------------------------------------------
function showWelcome {
cat <<Welcome-message
@@ -80,14 +82,10 @@ End-of-message
# -------------------------------------------------------
-# Configurable low water mark in GiB
-MINIMAL_MEMORY_GiB=2
-
function warnIfLowMemory {
- MINIMAL_MEMORY=$((MINIMAL_MEMORY_GiB*1024*1024)) # Convert to KiB
- INSTALLED_MEMORY=$(fgrep MemTotal /proc/meminfo | awk '{print $2}')
- if [ $((INSTALLED_MEMORY)) -le $((MINIMAL_MEMORY)) ];
- then
+ MINIMAL_MEMORY=2046755
+ INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
+ if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then
cat <<End-of-message
_ ___ ___
@@ -103,7 +101,8 @@ Your system is running on very little memory.
This means it may work but it wil most likely be slower than needed.
If you are running this via boot2docker you can simply increase
-the available memory to atleast ${MINIMAL_MEMORY_GiB} GiB (you have $((INSTALLED_MEMORY/(1024*1024))) GiB )
+the available memory to at least ${MINIMAL_MEMORY}KiB
+(you have ${INSTALLED_MEMORY}KiB )
End-of-message
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] hadoop git commit: HDFS-12637. Extend
TestDistributedFileSystemWithECFile with a random EC policy. Contributed by
Takanobu Asanuma.
Posted by ae...@apache.org.
HDFS-12637. Extend TestDistributedFileSystemWithECFile with a random EC policy. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bd70094
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bd70094
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bd70094
Branch: refs/heads/HDFS-7240
Commit: 7bd700941d0a423d5232331fa19eab5fdab6a6fb
Parents: 035c6ee
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Oct 16 09:54:37 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Oct 16 09:55:22 2017 -0700
----------------------------------------------------------------------
.../TestDistributedFileSystemWithECFile.java | 37 +++++++++------
...dFileSystemWithECFileWithRandomECPolicy.java | 49 ++++++++++++++++++++
2 files changed, 73 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd70094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
index d4e01b7..14a2ec4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
@@ -39,34 +39,45 @@ import static org.junit.Assert.assertTrue;
* FileSystem.listFiles for erasure coded files.
*/
public class TestDistributedFileSystemWithECFile {
- private final ErasureCodingPolicy ecPolicy =
- StripedFileTestUtil.getDefaultECPolicy();
- private final int cellSize = ecPolicy.getCellSize();
- private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
- private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
- private final int numDNs = dataBlocks + parityBlocks;
- private final int stripesPerBlock = 4;
- private final int blockSize = stripesPerBlock * cellSize;
- private final int blockGroupSize = blockSize * dataBlocks;
+ private ErasureCodingPolicy ecPolicy;
+ private int cellSize;
+ private short dataBlocks;
+ private short parityBlocks;
+ private int numDNs;
+ private int stripesPerBlock;
+ private int blockSize;
+ private int blockGroupSize;
private MiniDFSCluster cluster;
private FileContext fileContext;
private DistributedFileSystem fs;
private Configuration conf = new HdfsConfiguration();
+ public ErasureCodingPolicy getEcPolicy() {
+ return StripedFileTestUtil.getDefaultECPolicy();
+ }
+
@Before
public void setup() throws IOException {
+ ecPolicy = getEcPolicy();
+ cellSize = ecPolicy.getCellSize();
+ dataBlocks = (short) ecPolicy.getNumDataUnits();
+ parityBlocks = (short) ecPolicy.getNumParityUnits();
+ numDNs = dataBlocks + parityBlocks;
+ stripesPerBlock = 4;
+ blockSize = stripesPerBlock * cellSize;
+ blockGroupSize = blockSize * dataBlocks;
+
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
fileContext = FileContext.getFileContext(cluster.getURI(0), conf);
fs = cluster.getFileSystem();
- fs.enableErasureCodingPolicy(
- StripedFileTestUtil.getDefaultECPolicy().getName());
+ fs.enableErasureCodingPolicy(ecPolicy.getName());
fs.mkdirs(new Path("/ec"));
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
- StripedFileTestUtil.getDefaultECPolicy().getName());
+ ecPolicy.getName());
}
@After
@@ -121,7 +132,7 @@ public class TestDistributedFileSystemWithECFile {
@Test(timeout=60000)
public void testListECFilesSmallerThanOneStripe() throws Exception {
- int dataBlocksNum = 3;
+ int dataBlocksNum = dataBlocks;
createFile("/ec/smallstripe", cellSize * dataBlocksNum);
RemoteIterator<LocatedFileStatus> iter =
cluster.getFileSystem().listFiles(new Path("/ec"), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd70094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java
new file mode 100644
index 0000000..afa7569
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This test extends TestDistributedFileSystemWithECFile to use a random
+ * (non-default) EC policy.
+ */
+public class TestDistributedFileSystemWithECFileWithRandomECPolicy extends
+ TestDistributedFileSystemWithECFile {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestDistributedFileSystemWithECFileWithRandomECPolicy.class);
+
+ private ErasureCodingPolicy ecPolicy;
+
+ public TestDistributedFileSystemWithECFileWithRandomECPolicy() {
+ // If you want to debug this test with a specific ec policy, please use
+ // SystemErasureCodingPolicies class.
+ // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
+ ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
+ LOG.info("run {} with {}.",
+ TestDistributedFileSystemWithECFileWithRandomECPolicy.class
+ .getSuperclass().getSimpleName(), ecPolicy.getName());
+ }
+
+ @Override
+ public ErasureCodingPolicy getEcPolicy() {
+ return ecPolicy;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] hadoop git commit: HDFS-12612. DFSStripedOutputStream.close
will throw if called a second time with a failed streamer. (Lei (Eddy) Xu)
Posted by ae...@apache.org.
HDFS-12612. DFSStripedOutputStream.close will throw if called a second time with a failed streamer. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f27a4ad0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f27a4ad0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f27a4ad0
Branch: refs/heads/HDFS-7240
Commit: f27a4ad0324aa0b4080a1c4c6bf4cd560c927e20
Parents: 7532339
Author: Lei Xu <le...@apache.org>
Authored: Tue Oct 17 15:52:09 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Tue Oct 17 15:52:09 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/DFSStripedOutputStream.java | 40 +++++++----
.../org/apache/hadoop/hdfs/DataStreamer.java | 31 ++------
.../apache/hadoop/hdfs/ExceptionLastSeen.java | 75 +++++++++++++++++++
.../TestDFSStripedOutputStreamWithFailure.java | 76 ++++++++++++++++++++
4 files changed, 184 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f27a4ad0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 1b83959..39717ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -82,6 +82,12 @@ public class DFSStripedOutputStream extends DFSOutputStream
implements StreamCapabilities {
private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
+ /**
+ * OutputStream level last exception, will be used to indicate the fatal
+ * exception of this stream, i.e., being aborted.
+ */
+ private final ExceptionLastSeen exceptionLastSeen = new ExceptionLastSeen();
+
static class MultipleBlockingQueue<T> {
private final List<BlockingQueue<T>> queues;
@@ -971,12 +977,9 @@ public class DFSStripedOutputStream extends DFSOutputStream
if (isClosed()) {
return;
}
- for (StripedDataStreamer streamer : streamers) {
- streamer.getLastException().set(
- new IOException("Lease timeout of "
- + (dfsClient.getConf().getHdfsTimeout() / 1000)
- + " seconds expired."));
- }
+ exceptionLastSeen.set(new IOException("Lease timeout of "
+ + (dfsClient.getConf().getHdfsTimeout() / 1000)
+ + " seconds expired."));
try {
closeThreads(true);
@@ -1133,18 +1136,26 @@ public class DFSStripedOutputStream extends DFSOutputStream
@Override
protected synchronized void closeImpl() throws IOException {
if (isClosed()) {
+ exceptionLastSeen.check(true);
+
+ // Writing to at least {dataUnits} replicas can be considered as success,
+ // and the rest of data can be recovered.
+ final int minReplication = ecPolicy.getNumDataUnits();
+ int goodStreamers = 0;
final MultipleIOException.Builder b = new MultipleIOException.Builder();
- for(int i = 0; i < streamers.size(); i++) {
- final StripedDataStreamer si = getStripedDataStreamer(i);
+ for (final StripedDataStreamer si : streamers) {
try {
si.getLastException().check(true);
+ goodStreamers++;
} catch (IOException e) {
b.add(e);
}
}
- final IOException ioe = b.build();
- if (ioe != null) {
- throw ioe;
+ if (goodStreamers < minReplication) {
+ final IOException ioe = b.build();
+ if (ioe != null) {
+ throw ioe;
+ }
}
return;
}
@@ -1183,9 +1194,10 @@ public class DFSStripedOutputStream extends DFSOutputStream
}
} finally {
// Failures may happen when flushing data/parity data out. Exceptions
- // may be thrown if more than 3 streamers fail, or updatePipeline RPC
- // fails. Streamers may keep waiting for the new block/GS information.
- // Thus need to force closing these threads.
+ // may be thrown if the number of failed streamers is more than the
+ // number of parity blocks, or updatePipeline RPC fails. Streamers may
+ // keep waiting for the new block/GS information. Thus need to force
+ // closing these threads.
closeThreads(true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f27a4ad0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 99fa5f3..c1473dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -285,39 +285,22 @@ class DataStreamer extends Daemon {
packets.clear();
}
- class LastExceptionInStreamer {
- private IOException thrown;
-
- synchronized void set(Throwable t) {
- assert t != null;
- this.thrown = t instanceof IOException ?
- (IOException) t : new IOException(t);
- }
-
- synchronized void clear() {
- thrown = null;
- }
-
- /** Check if there already is an exception. */
+ class LastExceptionInStreamer extends ExceptionLastSeen {
+ /**
+ * Check if there already is an exception.
+ */
+ @Override
synchronized void check(boolean resetToNull) throws IOException {
+ final IOException thrown = get();
if (thrown != null) {
if (LOG.isTraceEnabled()) {
// wrap and print the exception to know when the check is called
LOG.trace("Got Exception while checking, " + DataStreamer.this,
new Throwable(thrown));
}
- final IOException e = thrown;
- if (resetToNull) {
- thrown = null;
- }
- throw e;
+ super.check(resetToNull);
}
}
-
- synchronized void throwException4Close() throws IOException {
- check(false);
- throw new ClosedChannelException();
- }
}
enum ErrorType {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f27a4ad0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExceptionLastSeen.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExceptionLastSeen.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExceptionLastSeen.java
new file mode 100644
index 0000000..06bc5d2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ExceptionLastSeen.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.IOException;
+import java.nio.channels.ClosedChannelException;
+
+/**
+ * The exception last seen by the {@link DataStreamer} or
+ * {@link DFSOutputStream}.
+ */
+@InterfaceAudience.Private
+class ExceptionLastSeen {
+ private IOException thrown;
+
+ /** Get the last seen exception. */
+ synchronized protected IOException get() {
+ return thrown;
+ }
+
+ /**
+ * Set the last seen exception.
+ * @param t the exception.
+ */
+ synchronized void set(Throwable t) {
+ assert t != null;
+ this.thrown = t instanceof IOException ?
+ (IOException) t : new IOException(t);
+ }
+
+ /** Clear the last seen exception. */
+ synchronized void clear() {
+ thrown = null;
+ }
+
+ /**
+ * Check if there already is an exception. Throw the exception if exist.
+ *
+ * @param resetToNull set to true to reset exception to null after calling
+ * this function.
+ * @throws IOException on existing IOException.
+ */
+ synchronized void check(boolean resetToNull) throws IOException {
+ if (thrown != null) {
+ final IOException e = thrown;
+ if (resetToNull) {
+ thrown = null;
+ }
+ throw e;
+ }
+ }
+
+ synchronized void throwException4Close() throws IOException {
+ check(false);
+ throw new ClosedChannelException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f27a4ad0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 57da439..e7fa278 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -319,6 +319,82 @@ public class TestDFSStripedOutputStreamWithFailure {
}
}
+ private void testCloseWithExceptionsInStreamer(
+ int numFailures, boolean shouldFail) throws Exception {
+ assertTrue(numFailures <=
+ ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits());
+ final Path dirFile = new Path(dir, "ecfile-" + numFailures);
+ try (FSDataOutputStream out = dfs.create(dirFile, true)) {
+ out.write("idempotent close".getBytes());
+
+ // Expect to raise IOE on the first close call, but any following
+ // close() should be no-op.
+ LambdaTestUtils.intercept(IOException.class,
+ out::close);
+
+ assertTrue(out.getWrappedStream() instanceof DFSStripedOutputStream);
+ DFSStripedOutputStream stripedOut =
+ (DFSStripedOutputStream) out.getWrappedStream();
+ for (int i = 0; i < numFailures; i++) {
+ // Only inject 1 stream failure.
+ stripedOut.getStripedDataStreamer(i).getLastException().set(
+ new IOException("injected failure")
+ );
+ }
+ if (shouldFail) {
+ LambdaTestUtils.intercept(IOException.class, out::close);
+ }
+
+ // Close multiple times. All the following close() should have no
+ // side-effect.
+ out.close();
+ }
+ }
+
+ // HDFS-12612
+ @Test
+ public void testIdempotentCloseWithFailedStreams() throws Exception {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ try {
+ setup(conf);
+ // shutdown few datanodes to avoid getting sufficient data blocks number
+ // of datanodes.
+ while (cluster.getDataNodes().size() >= dataBlocks) {
+ cluster.stopDataNode(0);
+ }
+ cluster.restartNameNodes();
+ cluster.triggerHeartbeats();
+
+ testCloseWithExceptionsInStreamer(1, false);
+ testCloseWithExceptionsInStreamer(ecPolicy.getNumParityUnits(), false);
+ testCloseWithExceptionsInStreamer(ecPolicy.getNumParityUnits() + 1, true);
+ testCloseWithExceptionsInStreamer(ecPolicy.getNumDataUnits(), true);
+ } finally {
+ tearDown();
+ }
+ }
+
+ @Test
+ public void testCloseAfterAbort() throws Exception {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ try {
+ setup(conf);
+
+ final Path dirFile = new Path(dir, "ecfile");
+ FSDataOutputStream out = dfs.create(dirFile, true);
+ assertTrue(out.getWrappedStream() instanceof DFSStripedOutputStream);
+ DFSStripedOutputStream stripedOut =
+ (DFSStripedOutputStream) out.getWrappedStream();
+ stripedOut.abort();
+ LambdaTestUtils.intercept(IOException.class,
+ "Lease timeout", stripedOut::close);
+ } finally {
+ tearDown();
+ }
+ }
+
@Test(timeout = 90000)
public void testAddBlockWhenNoSufficientParityNumOfNodes()
throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] hadoop git commit: YARN-7124. LogAggregationTFileController
deletes/renames while file is open. Contributed by Jason Lowe.
Posted by ae...@apache.org.
YARN-7124. LogAggregationTFileController deletes/renames while file is open. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1fcbe7cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1fcbe7cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1fcbe7cf
Branch: refs/heads/HDFS-7240
Commit: 1fcbe7cf5f2f7c609cd4158912f151848268ca9e
Parents: a50be1b
Author: Junping Du <ju...@apache.org>
Authored: Mon Oct 16 13:57:03 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Mon Oct 16 13:57:03 2017 -0700
----------------------------------------------------------------------
.../filecontroller/tfile/LogAggregationTFileController.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fcbe7cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
index 92e3a08..5064e26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
@@ -96,8 +96,10 @@ public class LogAggregationTFileController
@Override
public void closeWriter() {
- this.writer.close();
- this.writer = null;
+ if (this.writer != null) {
+ this.writer.close();
+ this.writer = null;
+ }
}
@Override
@@ -117,6 +119,9 @@ public class LogAggregationTFileController
record.increcleanupOldLogTimes();
}
+ // close the writer before the file is renamed or deleted
+ closeWriter();
+
final Path renamedPath = record.getRollingMonitorInterval() <= 0
? record.getRemoteNodeLogFileForApp() : new Path(
record.getRemoteNodeLogFileForApp().getParent(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] hadoop git commit: Revert "HADOOP-14954.
MetricsSystemImpl#init should increment refCount when already initialized.
Contributed by John Zhuge."
Posted by ae...@apache.org.
Revert "HADOOP-14954. MetricsSystemImpl#init should increment refCount when already initialized. Contributed by John Zhuge."
This reverts commit 14b3c2695b152285ed2919f2aa5a25e247c47871.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c904d60c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c904d60c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c904d60c
Branch: refs/heads/HDFS-7240
Commit: c904d60c38fe2720513a4824a56e9e8ebef6e17f
Parents: 0f2a691
Author: John Zhuge <jz...@apache.org>
Authored: Mon Oct 23 15:29:12 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Mon Oct 23 15:32:25 2017 -0700
----------------------------------------------------------------------
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 12 +----
.../metrics2/impl/TestMetricsSystemImpl.java | 46 --------------------
2 files changed, 1 insertion(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c904d60c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index c507a77..ee1672e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -140,16 +140,6 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
this(null);
}
- @VisibleForTesting
- boolean isMonitoring() {
- return monitoring;
- }
-
- @VisibleForTesting
- int getRefCount() {
- return refCount;
- }
-
/**
* Initialized the metrics system with a prefix.
* @param prefix the system will look for configs with the prefix
@@ -157,12 +147,12 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
*/
@Override
public synchronized MetricsSystem init(String prefix) {
- ++refCount;
if (monitoring && !DefaultMetricsSystem.inMiniClusterMode()) {
LOG.warn(this.prefix +" metrics system already initialized!");
return this;
}
this.prefix = checkNotNull(prefix, "prefix");
+ ++refCount;
if (monitoring) {
// in mini cluster mode
LOG.info(this.prefix +" metrics system started (again)");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c904d60c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
index 879de2d..abd1b13 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
@@ -26,8 +26,6 @@ import java.util.concurrent.atomic.*;
import javax.annotation.Nullable;
-import org.hamcrest.CoreMatchers;
-import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -561,50 +559,6 @@ public class TestMetricsSystemImpl {
ms.shutdown();
}
- @Test
- public void testInitShutdown() {
- boolean oldMode = DefaultMetricsSystem.inMiniClusterMode();
- try {
- DefaultMetricsSystem.setMiniClusterMode(true);
- runInitShutdownTests();
-
- DefaultMetricsSystem.setMiniClusterMode(false);
- runInitShutdownTests();
- } finally {
- DefaultMetricsSystem.setMiniClusterMode(oldMode);
- }
- }
-
- private void runInitShutdownTests() {
- MetricsSystemImpl ms = new MetricsSystemImpl();
- Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(false));
- Assert.assertThat(ms.getRefCount(), CoreMatchers.is(0));
-
- ms.init("TestMetricsSystem1");
- Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
- Assert.assertThat(ms.getRefCount(), CoreMatchers.is(1));
-
- ms.shutdown();
- Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(false));
- Assert.assertThat(ms.getRefCount(), CoreMatchers.is(0));
-
- ms.init("TestMetricsSystem2");
- Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
- Assert.assertThat(ms.getRefCount(), CoreMatchers.is(1));
-
- ms.init("TestMetricsSystem3");
- Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
- Assert.assertThat(ms.getRefCount(), CoreMatchers.is(2));
-
- ms.shutdown();
- Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(true));
- Assert.assertThat(ms.getRefCount(), CoreMatchers.is(1));
-
- ms.shutdown();
- Assert.assertThat(ms.isMonitoring(), CoreMatchers.is(false));
- Assert.assertThat(ms.getRefCount(), CoreMatchers.is(0));
- }
-
@Metrics(context="test")
private static class TestSource {
@Metric("C1 desc") MutableCounterLong c1;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] hadoop git commit: YARN-7338. Support same origin policy for
cross site scripting prevention. (Sunil G via wangda)
Posted by ae...@apache.org.
YARN-7338. Support same origin policy for cross site scripting prevention. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/298b174f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/298b174f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/298b174f
Branch: refs/heads/HDFS-7240
Commit: 298b174f663a06e67098f7b5cd645769c1a98a80
Parents: 3dd3d1d
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Oct 19 14:44:42 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Oct 19 14:44:42 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/webapp/WebApps.java | 22 ++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/298b174f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 9c5e8c3..4f1cacf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -401,6 +401,7 @@ public class WebApps {
WebApp webApp = build(webapp);
HttpServer2 httpServer = webApp.httpServer();
if (ui2Context != null) {
+ addFiltersForNewContext(ui2Context);
httpServer.addHandlerAtFront(ui2Context);
}
try {
@@ -413,6 +414,27 @@ public class WebApps {
return webApp;
}
+ private void addFiltersForNewContext(WebAppContext ui2Context) {
+ Map<String, String> params = getConfigParameters(csrfConfigPrefix);
+
+ if (hasCSRFEnabled(params)) {
+ LOG.info("CSRF Protection has been enabled for the {} application. "
+ + "Please ensure that there is an authentication mechanism "
+ + "enabled (kerberos, custom, etc).", name);
+ String restCsrfClassName = RestCsrfPreventionFilter.class.getName();
+ HttpServer2.defineFilter(ui2Context, restCsrfClassName,
+ restCsrfClassName, params, new String[]{"/*"});
+ }
+
+ params = getConfigParameters(xfsConfigPrefix);
+
+ if (hasXFSEnabled()) {
+ String xfsClassName = XFrameOptionsFilter.class.getName();
+ HttpServer2.defineFilter(ui2Context, xfsClassName, xfsClassName, params,
+ new String[]{"/*"});
+ }
+ }
+
private String inferHostClass() {
String thisClass = this.getClass().getName();
Throwable t = new Throwable();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] hadoop git commit: MAPREDUCE-6972. Enable try-with-resources
for RecordReader. Contributed by Zoltan Haindrich.
Posted by ae...@apache.org.
MAPREDUCE-6972. Enable try-with-resources for RecordReader. Contributed by Zoltan Haindrich.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2523e1cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2523e1cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2523e1cc
Branch: refs/heads/HDFS-7240
Commit: 2523e1cce5f6d6c3762c958c7f3138ab486c8497
Parents: 86ee0c5
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 18 11:18:03 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 18 11:18:39 2017 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapred/RecordReader.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2523e1cc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RecordReader.java
index 6e2c89f..0a996dc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RecordReader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapred;
+import java.io.Closeable;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -38,7 +39,7 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public interface RecordReader<K, V> {
+public interface RecordReader<K, V> extends Closeable{
/**
* Reads the next key/value pair from the input for processing.
*
@@ -74,7 +75,8 @@ public interface RecordReader<K, V> {
* Close this {@link InputSplit} to future operations.
*
* @throws IOException
- */
+ */
+ @Override
public void close() throws IOException;
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] hadoop git commit: YARN-7357. Several methods in
TestZKRMStateStore.TestZKRMStateStoreTester.TestZKRMStateStoreInternal should
have @Override annotations (Contributed by Sen Zhao via Daniel Templeton)
Posted by ae...@apache.org.
YARN-7357. Several methods in TestZKRMStateStore.TestZKRMStateStoreTester.TestZKRMStateStoreInternal should have @Override annotations
(Contributed by Sen Zhao via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e77dc2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e77dc2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e77dc2b
Branch: refs/heads/HDFS-7240
Commit: 9e77dc2bd149a38a99c0e2668ee27c91f99cf8a3
Parents: 96be795
Author: Daniel Templeton <te...@apache.org>
Authored: Mon Oct 23 13:49:30 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Mon Oct 23 13:51:19 2017 -0700
----------------------------------------------------------------------
.../server/resourcemanager/recovery/TestZKRMStateStore.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e77dc2b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index e5cf95d..f94bbd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -144,6 +144,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
return znodeWorkingPath + "/" + ROOT_ZNODE_NAME + "/" + VERSION_NODE;
}
+ @Override
public Version getCurrentVersion() {
return CURRENT_VERSION_INFO;
}
@@ -193,6 +194,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
return createStore(conf);
}
+ @Override
public RMStateStore getRMStateStore() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
return createStore(conf);
@@ -216,6 +218,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
return store.getCurrentVersion();
}
+ @Override
public boolean appExists(RMApp app) throws Exception {
String appIdPath = app.getApplicationId().toString();
int split =
@@ -225,6 +228,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
.forPath(store.getAppNode(appIdPath, split));
}
+ @Override
public boolean attemptExists(RMAppAttempt attempt) throws Exception {
ApplicationAttemptId attemptId = attempt.getAppAttemptId();
return null != curatorFramework.checkExists()
@@ -274,6 +278,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
static class TestAppRejDispatcher extends TestDispatcher {
private boolean appsavefailedEvnt;
+ @Override
public void handle(Event event) {
if (event instanceof RMAppEvent && event.getType()
.equals(RMAppEventType.APP_SAVE_FAILED)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] hadoop git commit: HDFS-12642. Log block and datanode details
in BlockRecoveryWorker.
Posted by ae...@apache.org.
HDFS-12642. Log block and datanode details in BlockRecoveryWorker.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21bc8555
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21bc8555
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21bc8555
Branch: refs/heads/HDFS-7240
Commit: 21bc85558718490e558c5b3bdb44c9c64eada994
Parents: 7bd7009
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Oct 16 10:33:16 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Oct 16 10:34:06 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/BlockRecoveryWorker.java | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/21bc8555/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
index 15c4af1..2ecd986 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
@@ -197,10 +197,9 @@ public class BlockRecoveryWorker {
long blockId = (isTruncateRecovery) ?
rBlock.getNewBlock().getBlockId() : block.getBlockId();
- if (LOG.isDebugEnabled()) {
- LOG.debug("block=" + block + ", (length=" + block.getNumBytes()
- + "), syncList=" + syncList);
- }
+ LOG.info("BlockRecoveryWorker: block={} (length={}),"
+ + " isTruncateRecovery={}, syncList={}", block,
+ block.getNumBytes(), isTruncateRecovery, syncList);
// syncList.isEmpty() means that all data-nodes do not have the block
// or their replicas have 0 length.
@@ -289,6 +288,11 @@ public class BlockRecoveryWorker {
newBlock.setNumBytes(rBlock.getNewBlock().getNumBytes());
}
+ LOG.info("BlockRecoveryWorker: block={} (length={}), bestState={},"
+ + " newBlock={} (length={}), participatingList={}",
+ block, block.getNumBytes(), bestState.name(), newBlock,
+ newBlock.getNumBytes(), participatingList);
+
List<DatanodeID> failedList = new ArrayList<>();
final List<BlockRecord> successList = new ArrayList<>();
for (BlockRecord r : participatingList) {
@@ -542,7 +546,7 @@ public class BlockRecoveryWorker {
ExtendedBlock block = rb.getBlock();
DatanodeInfo[] targets = rb.getLocations();
- LOG.info(who + " calls recoverBlock(" + block
+ LOG.info("BlockRecoveryWorker: " + who + " calls recoverBlock(" + block
+ ", targets=[" + Joiner.on(", ").join(targets) + "]"
+ ", newGenerationStamp=" + rb.getNewGenerationStamp()
+ ", newBlock=" + rb.getNewBlock()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] hadoop git commit: YARN-7261. Add debug message for better
download latency monitoring. (Yufei Gu)
Posted by ae...@apache.org.
YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0799fde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0799fde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0799fde3
Branch: refs/heads/HDFS-7240
Commit: 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1
Parents: b61144a
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Oct 20 10:00:13 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/util/FSDownload.java | 17 +++++++++++++++--
.../localizer/ResourceLocalizationService.java | 11 ++++++++++-
2 files changed, 25 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable<Path> {
} catch (URISyntaxException e) {
throw new IOException("Invalid resource", e);
}
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Starting to download " + sCopy);
+ }
+
createDir(destDirPath, cachePerms);
final Path dst_work = new Path(destDirPath + "_tmp");
createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable<Path> {
unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
changePermissions(dFinal.getFileSystem(conf), dFinal);
files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("File has been downloaded to " +
+ new Path(destDirPath, sCopy.getName()));
+ }
} catch (Exception e) {
try {
files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable<Path> {
// APPLICATION:
perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
}
- LOG.debug("Changing permissions for path " + path
- + " to perm " + perm);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+ }
+
final FsPermission fPerm = perm;
if (null == userUgi) {
files.setPermission(path, perm);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends CompositeService
// TODO handle failures, cancellation, requests by other containers
LocalizedResource rsrc = request.getResource();
LocalResourceRequest key = rsrc.getRequest();
- LOG.info("Downloading public rsrc:" + key);
+ LOG.info("Downloading public resource: " + key);
/*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends CompositeService
+ " Either queue is full or threadpool is shutdown.", re);
}
} else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip downloading resource: " + key + " since it's in"
+ + " state: " + rsrc.getState());
+ }
rsrc.unlock();
}
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip downloading resource: " + key + " since it is locked"
+ + " by other threads");
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] hadoop git commit: YARN-7308. TestApplicationACLs fails with
FairScheduler (rkanter)
Posted by ae...@apache.org.
YARN-7308. TestApplicationACLs fails with FairScheduler (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a615259
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a615259
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a615259
Branch: refs/heads/HDFS-7240
Commit: 8a615259281511de639c1f9aa719a842e34a523f
Parents: 4540ffd
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Oct 16 15:34:32 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon Oct 16 15:34:32 2017 -0700
----------------------------------------------------------------------
.../ParameterizedSchedulerTestBase.java | 3 +-
.../resourcemanager/TestApplicationACLs.java | 90 +++++++++++++++-----
2 files changed, 69 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a615259/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
index 289ff1c..9a29a89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
@@ -89,7 +89,8 @@ public abstract class ParameterizedSchedulerTestBase {
}
}
- private void configureFairScheduler(YarnConfiguration conf) throws IOException {
+ protected void configureFairScheduler(YarnConfiguration conf)
+ throws IOException {
// Disable queueMaxAMShare limitation for fair scheduler
PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a615259/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index cab1679..c7ed02c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -22,7 +22,10 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Matchers.any;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
@@ -30,6 +33,9 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.junit.After;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -57,19 +63,17 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-public class TestApplicationACLs {
+public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
private static final String APP_OWNER = "owner";
private static final String FRIEND = "friend";
@@ -82,27 +86,34 @@ public class TestApplicationACLs {
private static final Log LOG = LogFactory.getLog(TestApplicationACLs.class);
- static MockRM resourceManager;
- static Configuration conf = new YarnConfiguration();
- final static YarnRPC rpc = YarnRPC.create(conf);
- final static InetSocketAddress rmAddress = conf.getSocketAddr(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
- private static ApplicationClientProtocol rmClient;
+ private MockRM resourceManager;
+ private Configuration conf;
+ private YarnRPC rpc;
+ private InetSocketAddress rmAddress;
+ private ApplicationClientProtocol rmClient;
+ private RecordFactory recordFactory;
+ private boolean isQueueUser;
- private static RecordFactory recordFactory = RecordFactoryProvider
- .getRecordFactory(conf);
-
- private static boolean isQueueUser = false;
+ public TestApplicationACLs(SchedulerType type) throws IOException {
+ super(type);
+ }
- @BeforeClass
- public static void setup() throws InterruptedException, IOException {
- RMStateStore store = RMStateStoreFactory.getStore(conf);
+ @Before
+ public void setup() throws InterruptedException, IOException {
+ conf = getConf();
+ rpc = YarnRPC.create(conf);
+ rmAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ RMStateStoreFactory.getStore(conf);
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
AccessControlList adminACL = new AccessControlList("");
adminACL.addGroup(SUPER_GROUP);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
+ recordFactory = RecordFactoryProvider
+ .getRecordFactory(conf);
+ isQueueUser = false;
resourceManager = new MockRM(conf) {
@@ -161,13 +172,36 @@ public class TestApplicationACLs {
});
}
- @AfterClass
- public static void tearDown() {
+ @After
+ public void tearDown() {
if(resourceManager != null) {
resourceManager.stop();
}
}
+ @Override
+ protected void configureFairScheduler(YarnConfiguration conf)
+ throws IOException {
+ final String testDir = new File(System.getProperty("test.build.data",
+ "/tmp")).getAbsolutePath();
+ final String allocFile = new File(testDir, "test-queues.xml")
+ .getAbsolutePath();
+ PrintWriter out = new PrintWriter(new FileWriter(allocFile));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"root\" >");
+ out.println(" <queue name=\"default\">");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("<queuePlacementPolicy>");
+ out.println(" <rule name=\"specified\" create=\"false\" />");
+ out.println(" <rule name=\"reject\" />");
+ out.println("</queuePlacementPolicy>");
+ out.println("</allocations>");
+ out.close();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
+ }
+
@Test
public void testApplicationACLs() throws Exception {
@@ -207,6 +241,10 @@ public class TestApplicationACLs {
Resource resource = BuilderUtils.newResource(1024, 1);
context.setResource(resource);
amContainer.setApplicationACLs(acls);
+ if (conf.get(YarnConfiguration.RM_SCHEDULER)
+ .equals(FairScheduler.class.getName())) {
+ context.setQueue("root.default");
+ }
context.setAMContainerSpec(amContainer);
submitRequest.setApplicationSubmissionContext(context);
rmClient.submitApplication(submitRequest);
@@ -424,8 +462,14 @@ public class TestApplicationACLs {
GetApplicationReportResponse applicationReport =
rmClient.getApplicationReport(appReportRequest);
ApplicationReport appReport = applicationReport.getApplicationReport();
- Assert.assertTrue(appReport.getDiagnostics()
- .contains("submitted by user owner to unknown queue: InvalidQueue"));
+ if (conf.get(YarnConfiguration.RM_SCHEDULER)
+ .equals(FairScheduler.class.getName())) {
+ Assert.assertTrue(appReport.getDiagnostics()
+ .contains("Application rejected by queue placement policy"));
+ } else {
+ Assert.assertTrue(appReport.getDiagnostics()
+ .contains("submitted by user owner to unknown queue: InvalidQueue"));
+ }
}
private void verifyAdministerQueueUserAccess() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] hadoop git commit: YARN-7355. TestDistributedShell should be
scheduler agnostic. (Contributed by Haibo)
Posted by ae...@apache.org.
YARN-7355. TestDistributedShell should be scheduler agnostic. (Contributed by Haibo)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b7c87c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b7c87c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b7c87c9
Branch: refs/heads/HDFS-7240
Commit: 6b7c87c94592606966a4229313b3d0da48f16158
Parents: 0799fde
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Oct 20 11:15:20 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Oct 20 11:15:20 2017 -0700
----------------------------------------------------------------------
.../yarn/applications/distributedshell/TestDistributedShell.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b7c87c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index b541cae..0041c08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -149,7 +149,6 @@ public class TestDistributedShell {
conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
- conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
conf.set("mapreduce.jobhistory.address",
"0.0.0.0:" + ServerSocketUtil.getPort(10021, 10));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] hadoop git commit: YARN-4163: Audit getQueueInfo and
getApplications calls
Posted by ae...@apache.org.
YARN-4163: Audit getQueueInfo and getApplications calls
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/921338cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/921338cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/921338cd
Branch: refs/heads/HDFS-7240
Commit: 921338cd86e7215b0c4b1efdf2daf9449fb12c7b
Parents: 248d9b6
Author: Eric Payne <ep...@apache.org>
Authored: Mon Oct 23 11:43:41 2017 -0500
Committer: Eric Payne <ep...@apache.org>
Committed: Mon Oct 23 11:43:41 2017 -0500
----------------------------------------------------------------------
.../server/resourcemanager/ClientRMService.java | 16 ++
.../server/resourcemanager/RMAuditLogger.java | 172 +++++++++++++++++--
.../resourcemanager/TestRMAuditLogger.java | 47 ++++-
3 files changed, 217 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/921338cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index a0c3db6..e9c6eac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -150,6 +150,7 @@ import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -925,6 +926,8 @@ public class ClientRMService extends AbstractService implements
callerUGI.getUserName(), allowAccess));
}
+ RMAuditLogger.logSuccess(callerUGI.getUserName(),
+ AuditConstants.GET_APPLICATIONS_REQUEST, "ClientRMService");
GetApplicationsResponse response =
recordFactory.newRecordInstance(GetApplicationsResponse.class);
response.setApplicationList(reports);
@@ -975,6 +978,13 @@ public class ClientRMService extends AbstractService implements
GetQueueInfoResponse response =
recordFactory.newRecordInstance(GetQueueInfoResponse.class);
+ RMAuditLogger.ArgsBuilder arguments = new RMAuditLogger.ArgsBuilder()
+ .append(Keys.QUEUENAME, request.getQueueName())
+ .append(Keys.INCLUDEAPPS,
+ String.valueOf(request.getIncludeApplications()))
+ .append(Keys.INCLUDECHILDQUEUES,
+ String.valueOf(request.getIncludeChildQueues()))
+ .append(Keys.RECURSIVE, String.valueOf(request.getRecursive()));
try {
QueueInfo queueInfo =
scheduler.getQueueInfo(request.getQueueName(),
@@ -1001,8 +1011,14 @@ public class ClientRMService extends AbstractService implements
}
queueInfo.setApplications(appReports);
response.setQueueInfo(queueInfo);
+ RMAuditLogger.logSuccess(callerUGI.getUserName(),
+ AuditConstants.GET_QUEUE_INFO_REQUEST,
+ "ClientRMService", arguments);
} catch (IOException ioe) {
LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe);
+ RMAuditLogger.logFailure(callerUGI.getUserName(),
+ AuditConstants.GET_QUEUE_INFO_REQUEST, "UNKNOWN", "ClientRMService",
+ ioe.getMessage(), arguments);
}
return response;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/921338cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index 29ce636..ab10895 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -39,7 +39,8 @@ public class RMAuditLogger {
enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID,
- CALLERCONTEXT, CALLERSIGNATURE, RESOURCE}
+ CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUENAME,
+ INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE}
public static class AuditConstants {
static final String SUCCESS = "SUCCESS";
@@ -59,6 +60,10 @@ public class RMAuditLogger {
= "Get Application Attempt Report";
public static final String GET_CONTAINERS = "Get Containers";
public static final String GET_CONTAINER_REPORT = "Get Container Report";
+ public static final String GET_QUEUE_INFO_REQUEST =
+ "Get Queue Info Request";
+ public static final String GET_APPLICATIONS_REQUEST =
+ "Get Applications Request";
public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded";
public static final String FINISH_FAILED_APP = "Application Finished - Failed";
public static final String FINISH_KILLED_APP = "Application Finished - Killed";
@@ -97,12 +102,11 @@ public class RMAuditLogger {
}
/**
- * A helper api for creating an audit log for a successful event.
+ * A helper function for creating the common portion of a successful
+ * log message.
*/
- static String createSuccessLog(String user, String operation, String target,
- ApplicationId appId, ApplicationAttemptId attemptId,
- ContainerId containerId, Resource resource, CallerContext callerContext,
- InetAddress ip) {
+ private static StringBuilder createStringBuilderForSuccessEvent(String user,
+ String operation, String target, InetAddress ip) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
if (ip != null) {
@@ -111,6 +115,18 @@ public class RMAuditLogger {
add(Keys.OPERATION, operation, b);
add(Keys.TARGET, target ,b);
add(Keys.RESULT, AuditConstants.SUCCESS, b);
+ return b;
+ }
+
+ /**
+ * A helper api for creating an audit log for a successful event.
+ */
+ static String createSuccessLog(String user, String operation, String target,
+ ApplicationId appId, ApplicationAttemptId attemptId,
+ ContainerId containerId, Resource resource, CallerContext callerContext,
+ InetAddress ip) {
+ StringBuilder b =
+ createStringBuilderForSuccessEvent(user, operation, target, ip);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
@@ -151,6 +167,20 @@ public class RMAuditLogger {
}
/**
+ * A general helper api for creating an audit log for a successful event.
+ */
+ @SuppressWarnings("rawtypes")
+ static String createSuccessLog(String user, String operation, String target,
+ InetAddress ip, ArgsBuilder args) {
+ StringBuilder b =
+ createStringBuilderForSuccessEvent(user, operation, target, ip);
+ if(args != null) {
+ add(args, b);
+ }
+ return b.toString();
+ }
+
+ /**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request to the ResourceManager
@@ -173,11 +203,52 @@ public class RMAuditLogger {
}
/**
+ * Create a general readable and parseable audit log string for a successful
+ * event.
+ *
+ * @param user User who made the service request to the ResourceManager.
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param args The ArgsBuilder arguments for the operation request.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ * <br>
+ * This method will attempt to retrieve the remote IP
+ */
+ public static void logSuccess(String user, String operation, String target,
+ ArgsBuilder args) {
+ logSuccess(user, operation, target, Server.getRemoteIp(), args);
+ }
+
+ /**
+ * Create a general readable and parseable audit log string for a successful
+ * event.
+ *
+ * @param user User who made the service request to the ResourceManager.
+ * @param operation Operation requested by the user.
+ * @param target The target on which the operation is being performed.
+ * @param ip The ip address of the caller.
+ * @param args The ArgsBuilder arguments for the operation request.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logSuccess(String user, String operation, String target,
+ InetAddress ip, ArgsBuilder args) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, ip, args));
+ }
+ }
+
+ /**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request to the ResourceManager.
* @param operation Operation requested by the user.
- * @param target The target on which the operation is being performed.
+ * @param target The target on which the operation is being performed.
* @param appId Application Id in which operation was performed.
* @param attemptId Application Attempt Id in which operation was performed.
*
@@ -265,10 +336,8 @@ public class RMAuditLogger {
}
}
- static String createFailureLog(String user, String operation, String perm,
- String target, String description, ApplicationId appId,
- ApplicationAttemptId attemptId, ContainerId containerId,
- Resource resource, CallerContext callerContext) {
+ private static StringBuilder createStringBuilderForFailureLog(String user,
+ String operation, String target, String description, String perm) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
@@ -277,6 +346,18 @@ public class RMAuditLogger {
add(Keys.RESULT, AuditConstants.FAILURE, b);
add(Keys.DESCRIPTION, description, b);
add(Keys.PERMISSIONS, perm, b);
+ return b;
+ }
+
+ /**
+ * A helper api for creating an audit log for a failure event.
+ */
+ static String createFailureLog(String user, String operation, String perm,
+ String target, String description, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId,
+ Resource resource, CallerContext callerContext) {
+ StringBuilder b = createStringBuilderForFailureLog(user,
+ operation, target, description, perm);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
@@ -304,6 +385,20 @@ public class RMAuditLogger {
}
/**
+ * A helper api for creating an audit log for a failure event.
+ */
+ @SuppressWarnings("rawtypes")
+ static String createFailureLog(String user, String operation, String perm,
+ String target, String description, ArgsBuilder args) {
+ StringBuilder b = createStringBuilderForFailureLog(user,
+ operation, target, description, perm);
+ if(args != null) {
+ add(args, b);
+ }
+ return b.toString();
+ }
+
+ /**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
@@ -408,7 +503,29 @@ public class RMAuditLogger {
}
/**
- * A helper api to add remote IP address
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param perm Target permissions.
+ * @param target The target on which the operation is being performed.
+ * @param description The failure description
+ * @param args The arguments for the operation request.
+ *
+ * <br><br>
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+ * and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String perm,
+ String target, String description, ArgsBuilder args) {
+ if (LOG.isWarnEnabled()) {
+ LOG.warn(createFailureLog(user, operation, perm, target, description,
+ args));
+ }
+ }
+
+ /**
+ * A helper api to add remote IP address.
*/
static void addRemoteIP(StringBuilder b) {
InetAddress ip = Server.getRemoteIp();
@@ -434,4 +551,35 @@ public class RMAuditLogger {
b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
.append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
}
+
+ /**
+ * Appends the key-val pair to the passed builder in the following format
+ * <pair-delim>key=value
+ */
+ static void add(ArgsBuilder args, StringBuilder b) {
+ b.append(AuditConstants.PAIR_SEPARATOR).append(args.getArgs());
+ }
+
+ /**
+ * Builder to create and pass a list of arbitrary key value pairs for logging.
+ */
+ public static class ArgsBuilder {
+ private StringBuilder b;
+
+ public ArgsBuilder() {
+ b = new StringBuilder();
+ }
+
+ public ArgsBuilder append(Keys key, String value) {
+ if (b.length() != 0) {
+ b.append(AuditConstants.PAIR_SEPARATOR);
+ }
+ b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+ return this;
+ }
+
+ public StringBuilder getArgs() {
+ return b;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/921338cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index 3311f92..d5f1e35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -60,6 +60,7 @@ public class TestRMAuditLogger {
private static final String TARGET = "tgt";
private static final String PERM = "admin group";
private static final String DESC = "description of an audit log";
+ private static final String QUEUE = "root";
private static final ApplicationId APPID = mock(ApplicationId.class);
private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class);
private static final ContainerId CONTAINERID = mock(ContainerId.class);
@@ -123,14 +124,22 @@ public class TestRMAuditLogger {
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource, InetAddress remoteIp) {
+ testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId,
+ callerContext, resource, remoteIp, null);
+ }
+ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId,
+ CallerContext callerContext, Resource resource, InetAddress remoteIp,
+ RMAuditLogger.ArgsBuilder args) {
String sLog;
- if (checkIP) {
- sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
- attemptId, containerId, resource, callerContext, remoteIp);
+ InetAddress tmpIp = checkIP ? remoteIp : null;
+ if (args != null) {
+ sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
+ tmpIp, args);
} else {
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
- attemptId, containerId, resource, callerContext, null);
+ attemptId, containerId, resource, callerContext, tmpIp);
}
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
@@ -165,6 +174,10 @@ public class TestRMAuditLogger {
expLog.append("\tCALLERSIGNATURE=signature");
}
}
+ if (args != null) {
+ expLog.append("\tQUEUENAME=root");
+ expLog.append("\tRECURSIVE=true");
+ }
assertEquals(expLog.toString(), sLog);
}
@@ -241,6 +254,10 @@ public class TestRMAuditLogger {
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
.build(), RESOURCE);
+ RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder()
+ .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
+ testSuccessLogFormatHelper(checkIP, null, null, null, null, null,
+ Server.getRemoteIp(), args);
testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogNulls(checkIP);
}
@@ -256,9 +273,19 @@ public class TestRMAuditLogger {
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource) {
- String fLog =
+ testFailureLogFormatHelper(checkIP, appId, attemptId, containerId,
+ callerContext, resource, null);
+ }
+
+ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
+ ApplicationAttemptId attemptId, ContainerId containerId,
+ CallerContext callerContext, Resource resource,
+ RMAuditLogger.ArgsBuilder args) {
+ String fLog = args == null ?
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
- appId, attemptId, containerId, resource, callerContext);
+ appId, attemptId, containerId, resource, callerContext) :
+ RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
+ args);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
@@ -288,6 +315,10 @@ public class TestRMAuditLogger {
expLog.append("\tCALLERSIGNATURE=signature");
}
}
+ if (args != null) {
+ expLog.append("\tQUEUENAME=root");
+ expLog.append("\tRECURSIVE=true");
+ }
assertEquals(expLog.toString(), fLog);
}
@@ -314,6 +345,10 @@ public class TestRMAuditLogger {
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
.build(), RESOURCE);
+ RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder()
+ .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
+ testFailureLogFormatHelper(checkIP, null, null, null, null, null,
+ args);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] hadoop git commit: HADOOP-14949. TestKMS#testACLs fails
intermittently.
Posted by ae...@apache.org.
HADOOP-14949. TestKMS#testACLs fails intermittently.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7ff624c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7ff624c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7ff624c
Branch: refs/heads/HDFS-7240
Commit: b7ff624c767f76ca007d695afdc7a3815fceb04c
Parents: 1fcbe7c
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Oct 16 13:53:31 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Oct 16 14:19:31 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java | 4 ++++
.../apache/hadoop/crypto/key/kms/server/KMSConfiguration.java | 6 ++++++
.../java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++---
3 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7ff624c/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
index 096f756..b02f34e 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -326,4 +326,8 @@ public class KMSACLs implements Runnable, KeyACLs {
|| whitelistKeyAcls.containsKey(opType));
}
+ @VisibleForTesting
+ void forceNextReloadForTesting() {
+ lastReload = 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7ff624c/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index cf02dd1..df17ef5 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.crypto.key.kms.server;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.MalformedURLException;
@@ -31,6 +33,8 @@ import java.net.URL;
@InterfaceAudience.Private
public class KMSConfiguration {
+ static final Logger LOG = LoggerFactory.getLogger(KMSConfiguration.class);
+
public static final String KMS_CONFIG_DIR = "kms.config.dir";
public static final String KMS_DEFAULT_XML = "kms-default.xml";
public static final String KMS_SITE_XML = "kms-site.xml";
@@ -138,6 +142,8 @@ public class KMSConfiguration {
"' must be an absolute path: " + confDir);
}
File f = new File(confDir, KMS_ACLS_XML);
+ LOG.trace("Checking file {}, modification time is {}, last reload time is"
+ + " {}", f.getPath(), f.lastModified(), time);
// at least 100ms newer than time, we do this to ensure the file
// has been properly closed/flushed
newer = f.lastModified() - time > 100;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7ff624c/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 45546f2..30e84eb 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -54,6 +54,7 @@ import org.junit.Test;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
+import org.slf4j.event.Level;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1637,13 +1638,12 @@ public class TestKMS {
//stop the reloader, to avoid running while we are writing the new file
KMSWebApp.getACLs().stopReloader();
+ GenericTestUtils.setLogLevel(KMSConfiguration.LOG, Level.TRACE);
// test ACL reloading
- Thread.sleep(10); // to ensure the ACLs file modifiedTime is newer
conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "foo");
conf.set(KMSACLs.Type.GENERATE_EEK.getAclConfigKey(), "foo");
writeConf(testDir, conf);
- Thread.sleep(1000);
-
+ KMSWebApp.getACLs().forceNextReloadForTesting();
KMSWebApp.getACLs().run(); // forcing a reload by hand.
// should not be able to create a key now
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] hadoop git commit: YARN-7345. GPU Isolation: Incorrect minor
device numbers written to devices.deny file. (Jonathan Hung via wangda)
Posted by ae...@apache.org.
YARN-7345. GPU Isolation: Incorrect minor device numbers written to devices.deny file. (Jonathan Hung via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1b08ba7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1b08ba7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1b08ba7
Branch: refs/heads/HDFS-7240
Commit: c1b08ba720486e74461f0ec94a204c1ba4014c06
Parents: 298b174
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Oct 19 14:45:44 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Oct 19 14:45:44 2017 -0700
----------------------------------------------------------------------
.../container-executor/impl/modules/gpu/gpu-module.c | 2 +-
.../test/modules/gpu/test-gpu-module.cc | 13 +++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1b08ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
index f96645d..1a1b164 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
@@ -108,7 +108,7 @@ static int internal_handle_gpu_request(
char param_value[128];
memset(param_value, 0, sizeof(param_value));
snprintf(param_value, sizeof(param_value), "c %d:%d rwm",
- major_device_number, i);
+ major_device_number, minor_devices[i]);
int rc = update_cgroups_parameters_func_p("devices", "deny",
container_id, param_value);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1b08ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
index 7e41fb4..b3d93dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
@@ -165,6 +165,19 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
// Verify cgroups parameters
verify_param_updated_to_cgroups(0, NULL);
+
+ /* Test case 3: block 2 non-sequential devices */
+ cgroups_parameters_invoked.clear();
+ char* argv_2[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "1,3",
+ (char*) "--container_id", container_id };
+ rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 5, argv_2);
+ ASSERT_EQ(0, rc) << "Should success.\n";
+
+ // Verify cgroups parameters
+ const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 195:1 rwm",
+ "devices", "deny", container_id, "c 195:3 rwm"};
+ verify_param_updated_to_cgroups(8, expected_cgroups_argv_2);
}
TEST_F(TestGpuModule, test_illegal_cli_parameters) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] hadoop git commit: YARN-7170. Improve bower dependencies for
YARN UI v2. (Sunil G via wangda)
Posted by ae...@apache.org.
YARN-7170. Improve bower dependencies for YARN UI v2. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4afd308b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4afd308b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4afd308b
Branch: refs/heads/HDFS-7240
Commit: 4afd308b62d2335f31064c05bfefaf2294d874b0
Parents: ce7cf66
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Oct 19 21:58:31 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Oct 19 21:58:40 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 1 +
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc | 5 +----
3 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afd308b/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8980f0e..3baa8f3 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -138,6 +138,7 @@
<surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.11.199</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
+ <frontend-maven-plugin.version>1.5</frontend-maven-plugin.version>
<!-- the version of Hadoop declared in the version resources; can be overridden
so that Hadoop 3.x can declare itself a 2.x artifact. -->
<declared.hadoop.version>${project.version}</declared.hadoop.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afd308b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index 7aa03ed..b552f35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@ -132,7 +132,7 @@
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
- <version>1.2</version>
+ <version>${frontend-maven-plugin.version}</version>
<configuration>
<workingDirectory>${webappDir}</workingDirectory>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afd308b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
index 5b0b07d..959e169 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
@@ -1,7 +1,4 @@
{
"directory": "bower_components",
- "analytics": false,
- "resolvers": [
- "bower-shrinkwrap-resolver-ext"
- ]
+ "analytics": false
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] hadoop git commit: HDFS-12619. Do not catch and throw
unchecked exceptions if IBRs fail to process. Contributed by Wei-Chiu Chuang.
Posted by ae...@apache.org.
HDFS-12619. Do not catch and throw unchecked exceptions if IBRs fail to process. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ab0c8f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ab0c8f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ab0c8f9
Branch: refs/heads/HDFS-7240
Commit: 4ab0c8f96a41c573cc1f1e71c18871d243f952b9
Parents: 97c70c7
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Oct 19 06:17:59 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Oct 19 06:17:59 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/blockmanagement/BlockManager.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ab0c8f9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 386ba27..8b0f094 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3889,11 +3889,15 @@ public class BlockManager implements BlockStatsMXBean {
throw new IOException(
"Got incremental block report from unregistered or dead node");
}
+
+ boolean successful = false;
try {
processIncrementalBlockReport(node, srdb);
- } catch (Exception ex) {
- node.setForceRegistration(true);
- throw ex;
+ successful = true;
+ } finally {
+ if (!successful) {
+ node.setForceRegistration(true);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] hadoop git commit: YARN-6546. SLS is slow while loading 10k
queues. (Yufei Gu via Haibo Chen)
Posted by ae...@apache.org.
YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46eb1033
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46eb1033
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46eb1033
Branch: refs/heads/HDFS-7240
Commit: 46eb1033a86ca53e7b94202567aef8af2417bdf8
Parents: f27a4ad
Author: Haibo Chen <ha...@apache.org>
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Oct 17 16:04:19 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 1 +
.../sls/scheduler/CapacitySchedulerMetrics.java | 5 --
.../sls/scheduler/FairSchedulerMetrics.java | 17 +----
.../sls/scheduler/FifoSchedulerMetrics.java | 5 +-
.../sls/scheduler/SLSCapacityScheduler.java | 19 ------
.../yarn/sls/scheduler/SLSFairScheduler.java | 19 ------
.../yarn/sls/scheduler/SchedulerMetrics.java | 66 +++++++++-----------
7 files changed, 36 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
}
queueAppNumMap.put(queueName, appNum);
+ wrapper.getSchedulerMetrics().trackQueue(queueName);
}
private void runNewAM(String jobType, String user,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends SchedulerMetrics {
public CapacitySchedulerMetrics() {
super();
}
-
- @Override
- public void trackQueue(String queueName) {
- trackedQueues.add(queueName);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
@Override
- public void trackQueue(String queueName) {
- trackedQueues.add(queueName);
+ protected void registerQueueMetrics(String queueName) {
+ super.registerQueueMetrics(queueName);
+
FairScheduler fair = (FairScheduler) scheduler;
final FSQueue queue = fair.getQueueManager().getQueue(queueName);
registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
}
);
}
-
- @Override
- public void untrackQueue(String queueName) {
- trackedQueues.remove(queueName);
-
- for (Metric metric: Metric.values()) {
- metrics.remove("variable.queue." + queueName + "." +
- metric.value + ".memory");
- metrics.remove("variable.queue." + queueName + "." +
- metric.value + ".vcores");
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
index 6ab2e1d..427b597 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
@@ -35,8 +35,9 @@ public class FifoSchedulerMetrics extends SchedulerMetrics {
}
@Override
- public void trackQueue(String queueName) {
- trackedQueues.add(queueName);
+ protected void registerQueueMetrics(String queueName) {
+ super.registerQueueMetrics(queueName);
+
FifoScheduler fifo = (FifoScheduler) scheduler;
// for FifoScheduler, only DEFAULT_QUEUE
// here the three parameters doesn't affect results
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 108c2bc..6848b22 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -323,25 +323,6 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
queueName);
}
- private void initQueueMetrics(CSQueue queue) {
- if (queue instanceof LeafQueue) {
- schedulerMetrics.initQueueMetric(queue.getQueueName());
- return;
- }
-
- for (CSQueue child : queue.getChildQueues()) {
- initQueueMetrics(child);
- }
- }
- @Override
- public void serviceInit(Configuration configuration) throws Exception {
- super.serviceInit(configuration);
-
- if (metricsON) {
- initQueueMetrics(getRootQueue());
- }
- }
-
@Override
public void serviceStop() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 81f6648..8e49c51 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -306,25 +306,6 @@ public class SLSFairScheduler extends FairScheduler
queueName);
}
- private void initQueueMetrics(FSQueue queue) {
- if (queue instanceof FSLeafQueue) {
- schedulerMetrics.initQueueMetric(queue.getQueueName());
- return;
- }
-
- for (FSQueue child : queue.getChildQueues()) {
- initQueueMetrics(child);
- }
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- if (metricsON) {
- initQueueMetrics(getQueueManager().getRootQueue());
- }
- }
-
@Override
public void serviceStop() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
index cabc94a..9c4d000 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
@@ -234,14 +234,35 @@ public abstract class SchedulerMetrics {
}
}
- public abstract void trackQueue(String queueName);
-
- public void untrackQueue(String queueName) {
- for (String m : queueTrackedMetrics) {
- metrics.remove("variable.queue." + queueName + "." + m);
+ /**
+ * Track a queue by registering its metrics.
+ *
+ * @param queue queue name
+ */
+ public void trackQueue(String queue) {
+ queueLock.lock();
+ try {
+ if (!isTracked(queue)) {
+ trackedQueues.add(queue);
+ registerQueueMetrics(queue);
+ }
+ } finally {
+ queueLock.unlock();
}
}
-
+
+ protected void registerQueueMetrics(String queueName) {
+ SortedMap<String, Counter> counterMap = metrics.getCounters();
+
+ for (QueueMetric queueMetric : QueueMetric.values()) {
+ String metricName = getQueueMetricName(queueName, queueMetric);
+ if (!counterMap.containsKey(metricName)) {
+ metrics.counter(metricName);
+ queueTrackedMetrics.add(metricName);
+ }
+ }
+ }
+
public boolean isTracked(String queueName) {
return trackedQueues.contains(queueName);
}
@@ -547,40 +568,13 @@ public abstract class SchedulerMetrics {
return "counter.queue." + queue + "." + metric.value;
}
- private void traceQueueIfNotTraced(String queue) {
- queueLock.lock();
- try {
- if (!isTracked(queue)) {
- trackQueue(queue);
- }
- } finally {
- queueLock.unlock();
- }
- }
-
- void initQueueMetric(String queueName){
- SortedMap<String, Counter> counterMap = metrics.getCounters();
-
- for (QueueMetric queueMetric : QueueMetric.values()) {
- String metricName = getQueueMetricName(queueName, queueMetric);
- if (!counterMap.containsKey(metricName)) {
- metrics.counter(metricName);
- counterMap = metrics.getCounters();
- }
- }
-
- traceQueueIfNotTraced(queueName);
- }
-
void updateQueueMetrics(Resource pendingResource, Resource allocatedResource,
String queueName) {
+ trackQueue(queueName);
+
SortedMap<String, Counter> counterMap = metrics.getCounters();
for(QueueMetric metric : QueueMetric.values()) {
String metricName = getQueueMetricName(queueName, metric);
- if (!counterMap.containsKey(metricName)) {
- metrics.counter(metricName);
- counterMap = metrics.getCounters();
- }
if (metric == QueueMetric.PENDING_MEMORY) {
counterMap.get(metricName).inc(pendingResource.getMemorySize());
@@ -592,8 +586,6 @@ public abstract class SchedulerMetrics {
counterMap.get(metricName).inc(allocatedResource.getVirtualCores());
}
}
-
- traceQueueIfNotTraced(queueName);
}
void updateQueueMetricsByRelease(Resource releaseResource, String queue) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org