You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/12/05 21:55:44 UTC
[01/50] [abbrv] hadoop git commit: YARN-9030. Log aggregation changes
to handle filesystems which do not support setting permissions. (Suma
Shivaprasad via wangda)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 b5497070a -> 47d726015
YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)
Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9de8e8d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9de8e8d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9de8e8d0
Branch: refs/heads/HDFS-12943
Commit: 9de8e8d0496a2628b63cc841b1fdee80e2912f7a
Parents: 4d8de7a
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 21 17:28:37 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 13:36:21 2018 -0800
----------------------------------------------------------------------
.../LogAggregationFileController.java | 74 +++++++++++++++-----
1 file changed, 56 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de8e8d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index fe65288..e37308d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -109,6 +109,8 @@ public abstract class LogAggregationFileController {
protected int retentionSize;
protected String fileControllerName;
+ protected boolean fsSupportsChmod = true;
+
public LogAggregationFileController() {}
/**
@@ -250,7 +252,6 @@ public abstract class LogAggregationFileController {
* Verify and create the remote log directory.
*/
public void verifyAndCreateRemoteLogDir() {
- boolean logPermError = true;
// Checking the existence of the TLD
FileSystem remoteFS = null;
try {
@@ -264,14 +265,12 @@ public abstract class LogAggregationFileController {
try {
FsPermission perms =
remoteFS.getFileStatus(remoteRootLogDir).getPermission();
- if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) {
+ if (!perms.equals(TLDIR_PERMISSIONS)) {
LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
+ "] already exist, but with incorrect permissions. "
+ "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
+ "]." + " The cluster may have problems with multiple users.");
- logPermError = false;
- } else {
- logPermError = true;
+
}
} catch (FileNotFoundException e) {
remoteExists = false;
@@ -280,15 +279,26 @@ public abstract class LogAggregationFileController {
"Failed to check permissions for dir ["
+ remoteRootLogDir + "]", e);
}
+
+ Path qualified =
+ remoteRootLogDir.makeQualified(remoteFS.getUri(),
+ remoteFS.getWorkingDirectory());
if (!remoteExists) {
LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
+ "] does not exist. Attempting to create it.");
try {
- Path qualified =
- remoteRootLogDir.makeQualified(remoteFS.getUri(),
- remoteFS.getWorkingDirectory());
remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
- remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+ // Not possible to query FileSystem API to check if it supports
+ // chmod, chown etc. Hence resorting to catching exceptions here.
+ // Remove when FS APi is ready
+ try {
+ remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+ } catch ( UnsupportedOperationException use) {
+ LOG.info("Unable to set permissions for configured filesystem since"
+ + " it does not support this", remoteFS.getScheme());
+ fsSupportsChmod = false;
+ }
UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
String primaryGroupName = null;
@@ -301,13 +311,31 @@ public abstract class LogAggregationFileController {
}
// set owner on the remote directory only if the primary group exists
if (primaryGroupName != null) {
- remoteFS.setOwner(qualified,
- loginUser.getShortUserName(), primaryGroupName);
+ try {
+ remoteFS.setOwner(qualified, loginUser.getShortUserName(),
+ primaryGroupName);
+ } catch (UnsupportedOperationException use) {
+ LOG.info(
+ "File System does not support setting user/group" + remoteFS
+ .getScheme(), use);
+ }
}
} catch (IOException e) {
throw new YarnRuntimeException("Failed to create remoteLogDir ["
+ remoteRootLogDir + "]", e);
}
+ } else{
+ //Check if FS has capability to set/modify permissions
+ try {
+ remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+ } catch (UnsupportedOperationException use) {
+ LOG.info("Unable to set permissions for configured filesystem since"
+ + " it does not support this", remoteFS.getScheme());
+ fsSupportsChmod = false;
+ } catch (IOException e) {
+ LOG.warn("Failed to check if FileSystem suppports permissions on "
+ + "remoteLogDir [" + remoteRootLogDir + "]", e);
+ }
}
}
@@ -383,11 +411,16 @@ public abstract class LogAggregationFileController {
protected void createDir(FileSystem fs, Path path, FsPermission fsPerm)
throws IOException {
- FsPermission dirPerm = new FsPermission(fsPerm);
- fs.mkdirs(path, dirPerm);
- FsPermission umask = FsPermission.getUMask(fs.getConf());
- if (!dirPerm.equals(dirPerm.applyUMask(umask))) {
- fs.setPermission(path, new FsPermission(fsPerm));
+
+ if (fsSupportsChmod) {
+ FsPermission dirPerm = new FsPermission(fsPerm);
+ fs.mkdirs(path, dirPerm);
+ FsPermission umask = FsPermission.getUMask(fs.getConf());
+ if (!dirPerm.equals(dirPerm.applyUMask(umask))) {
+ fs.setPermission(path, new FsPermission(fsPerm));
+ }
+ } else {
+ fs.mkdirs(path);
}
}
@@ -396,8 +429,10 @@ public abstract class LogAggregationFileController {
boolean exists = true;
try {
FileStatus appDirStatus = fs.getFileStatus(path);
- if (!APP_DIR_PERMISSIONS.equals(appDirStatus.getPermission())) {
- fs.setPermission(path, APP_DIR_PERMISSIONS);
+ if (fsSupportsChmod) {
+ if (!APP_DIR_PERMISSIONS.equals(appDirStatus.getPermission())) {
+ fs.setPermission(path, APP_DIR_PERMISSIONS);
+ }
}
} catch (FileNotFoundException fnfe) {
exists = false;
@@ -502,4 +537,7 @@ public abstract class LogAggregationFileController {
return sb.toString();
}
+ public boolean isFsSupportsChmod() {
+ return fsSupportsChmod;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HADOOP-15852. Refactor QuotaUsage.
Contributed by Beluga Behr.
Posted by sh...@apache.org.
HADOOP-15852. Refactor QuotaUsage. Contributed by Beluga Behr.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb9deed4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb9deed4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb9deed4
Branch: refs/heads/HDFS-12943
Commit: fb9deed41d6b9f242474b474a5acde0c858e28f6
Parents: 69489ff
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Mon Dec 3 13:25:02 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Mon Dec 3 13:25:02 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/QuotaUsage.java | 108 ++++++++++---------
1 file changed, 55 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb9deed4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
index 3472362..4e42e5b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -40,14 +40,12 @@ public class QuotaUsage {
/** Builder class for QuotaUsage. */
public static class Builder {
public Builder() {
- this.quota = -1;
- this.spaceQuota = -1;
+ this.quota = -1L;
+ this.spaceQuota = -1L;
typeConsumed = new long[StorageType.values().length];
typeQuota = new long[StorageType.values().length];
- for (int i = 0; i < typeQuota.length; i++) {
- typeQuota[i] = -1;
- }
+ Arrays.fill(typeQuota, -1L);
}
public Builder fileAndDirectoryCount(long count) {
@@ -71,9 +69,8 @@ public class QuotaUsage {
}
public Builder typeConsumed(long[] typeConsumed) {
- for (int i = 0; i < typeConsumed.length; i++) {
- this.typeConsumed[i] = typeConsumed[i];
- }
+ System.arraycopy(typeConsumed, 0, this.typeConsumed, 0,
+ typeConsumed.length);
return this;
}
@@ -88,9 +85,8 @@ public class QuotaUsage {
}
public Builder typeQuota(long[] typeQuota) {
- for (int i = 0; i < typeQuota.length; i++) {
- this.typeQuota[i] = typeQuota[i];
- }
+ System.arraycopy(typeQuota, 0, this.typeQuota, 0,
+ typeQuota.length);
return this;
}
@@ -153,22 +149,12 @@ public class QuotaUsage {
/** Return storage type quota. */
public long getTypeQuota(StorageType type) {
- return (typeQuota != null) ? typeQuota[type.ordinal()] : -1;
+ return (typeQuota != null) ? typeQuota[type.ordinal()] : -1L;
}
/** Return storage type consumed. */
public long getTypeConsumed(StorageType type) {
- return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0;
- }
-
- /** Return storage type quota. */
- private long[] getTypesQuota() {
- return typeQuota;
- }
-
- /** Return storage type quota. */
- private long[] getTypesConsumed() {
- return typeConsumed;
+ return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0L;
}
/** Return true if any storage type quota has been set. */
@@ -177,7 +163,7 @@ public class QuotaUsage {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeQuota[t.ordinal()] > 0) {
+ if (typeQuota[t.ordinal()] > 0L) {
return true;
}
}
@@ -190,7 +176,7 @@ public class QuotaUsage {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeConsumed[t.ordinal()] > 0) {
+ if (typeConsumed[t.ordinal()] > 0L) {
return true;
}
}
@@ -198,33 +184,50 @@ public class QuotaUsage {
}
@Override
- public boolean equals(Object to) {
- return (this == to || (to instanceof QuotaUsage &&
- getFileAndDirectoryCount() ==
- ((QuotaUsage) to).getFileAndDirectoryCount() &&
- getQuota() == ((QuotaUsage) to).getQuota() &&
- getSpaceConsumed() == ((QuotaUsage) to).getSpaceConsumed() &&
- getSpaceQuota() == ((QuotaUsage) to).getSpaceQuota() &&
- Arrays.equals(getTypesQuota(), ((QuotaUsage) to).getTypesQuota()) &&
- Arrays.equals(getTypesConsumed(),
- ((QuotaUsage) to).getTypesConsumed())));
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + (int) (fileAndDirectoryCount ^ (fileAndDirectoryCount >>> 32));
+ result = prime * result + (int) (quota ^ (quota >>> 32));
+ result = prime * result + (int) (spaceConsumed ^ (spaceConsumed >>> 32));
+ result = prime * result + (int) (spaceQuota ^ (spaceQuota >>> 32));
+ result = prime * result + Arrays.hashCode(typeConsumed);
+ result = prime * result + Arrays.hashCode(typeQuota);
+ return result;
}
@Override
- public int hashCode() {
- long result = (getFileAndDirectoryCount() ^ getQuota() ^
- getSpaceConsumed() ^ getSpaceQuota());
- if (getTypesQuota() != null) {
- for (long quota : getTypesQuota()) {
- result ^= quota;
- }
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
}
- if (getTypesConsumed() != null) {
- for (long consumed : getTypesConsumed()) {
- result ^= consumed;
- }
+ if (obj == null) {
+ return false;
}
- return (int)result;
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ QuotaUsage other = (QuotaUsage) obj;
+ if (fileAndDirectoryCount != other.fileAndDirectoryCount) {
+ return false;
+ }
+ if (quota != other.quota) {
+ return false;
+ }
+ if (spaceConsumed != other.spaceConsumed) {
+ return false;
+ }
+ if (spaceQuota != other.spaceQuota) {
+ return false;
+ }
+ if (!Arrays.equals(typeConsumed, other.typeConsumed)) {
+ return false;
+ }
+ if (!Arrays.equals(typeQuota, other.typeQuota)) {
+ return false;
+ }
+ return true;
}
/**
@@ -292,11 +295,11 @@ public class QuotaUsage {
String spaceQuotaStr = QUOTA_NONE;
String spaceQuotaRem = QUOTA_INF;
- if (quota > 0) {
+ if (quota > 0L) {
quotaStr = formatSize(quota, hOption);
quotaRem = formatSize(quota-fileAndDirectoryCount, hOption);
}
- if (spaceQuota >= 0) {
+ if (spaceQuota >= 0L) {
spaceQuotaStr = formatSize(spaceQuota, hOption);
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
@@ -307,7 +310,7 @@ public class QuotaUsage {
protected String getTypesQuotaUsage(boolean hOption,
List<StorageType> types) {
- StringBuffer content = new StringBuffer();
+ StringBuilder content = new StringBuilder();
for (StorageType st : types) {
long typeQuota = getTypeQuota(st);
long typeConsumed = getTypeConsumed(st);
@@ -332,13 +335,12 @@ public class QuotaUsage {
* @return storage header string
*/
public static String getStorageTypeHeader(List<StorageType> storageTypes) {
- StringBuffer header = new StringBuffer();
-
+ StringBuilder header = new StringBuilder();
for (StorageType st : storageTypes) {
/* the field length is 13/17 for quota and remain quota
* as the max length for quota name is ARCHIVE_QUOTA
* and remain quota name REM_ARCHIVE_QUOTA */
- String storageName = st.toString();
+ final String storageName = st.toString();
header.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT,
storageName + "_QUOTA", "REM_" + storageName + "_QUOTA"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDDS-877. Ensure correct surefire
version for Ozone test. Contributed by Xiaoyu Yao.
Posted by sh...@apache.org.
HDDS-877. Ensure correct surefire version for Ozone test. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae5fbdd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae5fbdd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae5fbdd9
Branch: refs/heads/HDFS-12943
Commit: ae5fbdd9ed6ef09b588637f2eadd7a04e8382289
Parents: f534736
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Nov 29 11:37:36 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Nov 29 11:37:36 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/pom.xml | 1 +
hadoop-ozone/pom.xml | 3 +--
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5fbdd9/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 869ecbf..5537b3a 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -53,6 +53,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<protobuf-maven-plugin.version>0.5.1</protobuf-maven-plugin.version>
<os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+ <maven-surefire-plugin.version>3.0.0-M1</maven-surefire-plugin.version>
</properties>
<repositories>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae5fbdd9/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 39c65d5..4c13bd6 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -37,8 +37,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<bouncycastle.version>1.60</bouncycastle.version>
<ozone.release>Badlands</ozone.release>
<declared.ozone.version>${ozone.version}</declared.ozone.version>
-
-
+ <maven-surefire-plugin.version>3.0.0-M1</maven-surefire-plugin.version>
</properties>
<modules>
<module>common</module>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-8948. PlacementRule interface
should be for all YarnSchedulers. Contributed by Bibin A Chundatt.
Posted by sh...@apache.org.
YARN-8948. PlacementRule interface should be for all YarnSchedulers. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a68d766e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a68d766e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a68d766e
Branch: refs/heads/HDFS-12943
Commit: a68d766e876631d7ee2e1a6504d4120ba628d178
Parents: c1d24f8
Author: bibinchundatt <bi...@apache.org>
Authored: Thu Nov 29 21:43:34 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Thu Nov 29 21:43:34 2018 +0530
----------------------------------------------------------------------
.../placement/AppNameMappingPlacementRule.java | 12 ++++++++++--
.../server/resourcemanager/placement/PlacementRule.java | 4 ++--
.../placement/UserGroupMappingPlacementRule.java | 11 ++++++++++-
3 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
index 2debade..7a46962 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
@@ -20,11 +20,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
@@ -61,8 +62,15 @@ public class AppNameMappingPlacementRule extends PlacementRule {
}
@Override
- public boolean initialize(CapacitySchedulerContext schedulerContext)
+ public boolean initialize(ResourceScheduler scheduler)
throws IOException {
+ if (!(scheduler instanceof CapacityScheduler)) {
+ throw new IOException(
+ "AppNameMappingPlacementRule can be configured only for "
+ + "CapacityScheduler");
+ }
+ CapacitySchedulerContext schedulerContext =
+ (CapacitySchedulerContext) scheduler;
CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
LOG.info(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
index 21ab32a..0f3d43c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
public abstract class PlacementRule {
@@ -31,7 +31,7 @@ public abstract class PlacementRule {
}
public abstract boolean initialize(
- CapacitySchedulerContext schedulerContext) throws IOException;
+ ResourceScheduler scheduler) throws IOException;
/**
* Get queue for a given application
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a68d766e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 99f37b1..a9e8235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -34,8 +34,10 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
@@ -230,8 +232,15 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
@VisibleForTesting
@Override
- public boolean initialize(CapacitySchedulerContext schedulerContext)
+ public boolean initialize(ResourceScheduler scheduler)
throws IOException {
+ if (!(scheduler instanceof CapacityScheduler)) {
+ throw new IOException(
+ "UserGroupMappingPlacementRule can be configured only for "
+ + "CapacityScheduler");
+ }
+ CapacitySchedulerContext schedulerContext =
+ (CapacitySchedulerContext) scheduler;
CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
LOG.info(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDFS-14095. EC: Track Erasure
Coding commands in DFS statistics. Contributed by Ayush Saxena.
Posted by sh...@apache.org.
HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5347368
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5347368
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5347368
Branch: refs/heads/HDFS-12943
Commit: f534736867eed962899615ca1b7eb68bcf591d17
Parents: d0edd37
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Fri Nov 30 00:18:27 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Nov 30 00:18:27 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/DFSOpsCountStatistics.java | 9 +++
.../hadoop/hdfs/DistributedFileSystem.java | 18 ++++++
.../hadoop/hdfs/TestDistributedFileSystem.java | 63 +++++++++++++++++++-
3 files changed, 89 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
index 3dcf13b..b9852ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
@@ -41,6 +41,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
/** This is for counting distributed file system operations. */
public enum OpType {
+ ADD_EC_POLICY("op_add_ec_policy"),
ALLOW_SNAPSHOT("op_allow_snapshot"),
APPEND(CommonStatisticNames.OP_APPEND),
CONCAT("op_concat"),
@@ -51,10 +52,15 @@ public class DFSOpsCountStatistics extends StorageStatistics {
CREATE_SYM_LINK("op_create_symlink"),
DELETE(CommonStatisticNames.OP_DELETE),
DELETE_SNAPSHOT("op_delete_snapshot"),
+ DISABLE_EC_POLICY("op_disable_ec_policy"),
DISALLOW_SNAPSHOT("op_disallow_snapshot"),
+ ENABLE_EC_POLICY("op_enable_ec_policy"),
EXISTS(CommonStatisticNames.OP_EXISTS),
GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
+ GET_EC_CODECS("op_get_ec_codecs"),
+ GET_EC_POLICY("op_get_ec_policy"),
+ GET_EC_POLICIES("op_get_ec_policies"),
GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
GET_FILE_LINK_STATUS("op_get_file_link_status"),
@@ -76,11 +82,13 @@ public class DFSOpsCountStatistics extends StorageStatistics {
REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
+ REMOVE_EC_POLICY("op_remove_ec_policy"),
REMOVE_XATTR("op_remove_xattr"),
RENAME(CommonStatisticNames.OP_RENAME),
RENAME_SNAPSHOT("op_rename_snapshot"),
RESOLVE_LINK("op_resolve_link"),
SET_ACL(CommonStatisticNames.OP_SET_ACL),
+ SET_EC_POLICY("op_set_ec_policy"),
SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
SET_REPLICATION("op_set_replication"),
@@ -90,6 +98,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
GET_SNAPSHOT_DIFF("op_get_snapshot_diff"),
GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"),
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
+ UNSET_EC_POLICY("op_unset_ec_policy"),
UNSET_STORAGE_POLICY("op_unset_storage_policy");
private static final Map<String, OpType> SYMBOL_MAP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index ca1546c..7dd02bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2845,6 +2845,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void setErasureCodingPolicy(final Path path,
final String ecPolicyName) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.SET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
@@ -2912,6 +2914,8 @@ public class DistributedFileSystem extends FileSystem
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_POLICY);
Path absF = fixRelativePart(path);
return new FileSystemLinkResolver<ErasureCodingPolicy>() {
@Override
@@ -2943,6 +2947,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_POLICIES);
return Arrays.asList(dfs.getErasureCodingPolicies());
}
@@ -2955,6 +2961,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Map<String, String> getAllErasureCodingCodecs()
throws IOException {
+ statistics.incrementReadOps(1);
+ storageStatistics.incrementOpCounter(OpType.GET_EC_CODECS);
return dfs.getErasureCodingCodecs();
}
@@ -2971,6 +2979,8 @@ public class DistributedFileSystem extends FileSystem
*/
public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
ErasureCodingPolicy[] policies) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.ADD_EC_POLICY);
return dfs.addErasureCodingPolicies(policies);
}
@@ -2982,6 +2992,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.REMOVE_EC_POLICY);
dfs.removeErasureCodingPolicy(ecPolicyName);
}
@@ -2993,6 +3005,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void enableErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.ENABLE_EC_POLICY);
dfs.enableErasureCodingPolicy(ecPolicyName);
}
@@ -3004,6 +3018,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void disableErasureCodingPolicy(String ecPolicyName)
throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.DISABLE_EC_POLICY);
dfs.disableErasureCodingPolicy(ecPolicyName);
}
@@ -3014,6 +3030,8 @@ public class DistributedFileSystem extends FileSystem
* @throws IOException
*/
public void unsetErasureCodingPolicy(final Path path) throws IOException {
+ statistics.incrementWriteOps(1);
+ storageStatistics.incrementOpCounter(OpType.UNSET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index cae0fbf..97ced7a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -773,7 +773,68 @@ public class TestDistributedFileSystem {
} finally {
if (cluster != null) cluster.shutdown();
}
-
+ }
+
+ @Test
+ public void testECStatistics() throws IOException {
+ try (MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(getTestConfiguration()).build()) {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ Path dir = new Path("/test");
+ dfs.mkdirs(dir);
+ int readOps = 0;
+ int writeOps = 0;
+ FileSystem.clearStatistics();
+
+ long opCount = getOpStatistics(OpType.ENABLE_EC_POLICY);
+ dfs.enableErasureCodingPolicy("RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.ENABLE_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.SET_EC_POLICY);
+ dfs.setErasureCodingPolicy(dir, "RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.SET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_POLICY);
+ dfs.getErasureCodingPolicy(dir);
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.UNSET_EC_POLICY);
+ dfs.unsetErasureCodingPolicy(dir);
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.UNSET_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_POLICIES);
+ dfs.getAllErasureCodingPolicies();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_POLICIES, opCount + 1);
+
+ opCount = getOpStatistics(OpType.GET_EC_CODECS);
+ dfs.getAllErasureCodingCodecs();
+ checkStatistics(dfs, ++readOps, writeOps, 0);
+ checkOpStatistics(OpType.GET_EC_CODECS, opCount + 1);
+
+ ErasureCodingPolicy newPolicy =
+ new ErasureCodingPolicy(new ECSchema("rs", 5, 3), 1024 * 1024);
+
+ opCount = getOpStatistics(OpType.ADD_EC_POLICY);
+ dfs.addErasureCodingPolicies(new ErasureCodingPolicy[] {newPolicy});
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.ADD_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.REMOVE_EC_POLICY);
+ dfs.removeErasureCodingPolicy("RS-5-3-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.REMOVE_EC_POLICY, opCount + 1);
+
+ opCount = getOpStatistics(OpType.DISABLE_EC_POLICY);
+ dfs.disableErasureCodingPolicy("RS-10-4-1024k");
+ checkStatistics(dfs, readOps, ++writeOps, 0);
+ checkOpStatistics(OpType.DISABLE_EC_POLICY, opCount + 1);
+ }
}
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDDS-808. Simplify OMAction and
DNAction classes used for AuditLogging. Contributed by Dinesh Chitlangia.
Posted by sh...@apache.org.
HDDS-808. Simplify OMAction and DNAction classes used for AuditLogging. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/184cced5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/184cced5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/184cced5
Branch: refs/heads/HDFS-12943
Commit: 184cced513c5599d7b33c9124692fbcd2e6d338e
Parents: 07142f5
Author: Ajay Kumar <aj...@apache.org>
Authored: Thu Nov 29 08:35:02 2018 -0800
Committer: Ajay Kumar <aj...@apache.org>
Committed: Thu Nov 29 08:35:20 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/audit/DNAction.java | 44 +++++++---------
.../apache/hadoop/ozone/audit/DummyAction.java | 36 ++++++-------
.../org/apache/hadoop/ozone/audit/OMAction.java | 54 +++++++++-----------
3 files changed, 58 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
index ce34c46..1c87f2b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/DNAction.java
@@ -21,34 +21,28 @@ package org.apache.hadoop.ozone.audit;
*/
public enum DNAction implements AuditAction {
- CREATE_CONTAINER("CREATE_CONTAINER"),
- READ_CONTAINER("READ_CONTAINER"),
- UPDATE_CONTAINER("UPDATE_CONTAINER"),
- DELETE_CONTAINER("DELETE_CONTAINER"),
- LIST_CONTAINER("LIST_CONTAINER"),
- PUT_BLOCK("PUT_BLOCK"),
- GET_BLOCK("GET_BLOCK"),
- DELETE_BLOCK("DELETE_BLOCK"),
- LIST_BLOCK("LIST_BLOCK"),
- READ_CHUNK("READ_CHUNK"),
- DELETE_CHUNK("DELETE_CHUNK"),
- WRITE_CHUNK("WRITE_CHUNK"),
- LIST_CHUNK("LIST_CHUNK"),
- COMPACT_CHUNK("COMPACT_CHUNK"),
- PUT_SMALL_FILE("PUT_SMALL_FILE"),
- GET_SMALL_FILE("GET_SMALL_FILE"),
- CLOSE_CONTAINER("CLOSE_CONTAINER"),
- GET_COMMITTED_BLOCK_LENGTH("GET_COMMITTED_BLOCK_LENGTH");
-
- private String action;
-
- DNAction(String action) {
- this.action = action;
- }
+ CREATE_CONTAINER,
+ READ_CONTAINER,
+ UPDATE_CONTAINER,
+ DELETE_CONTAINER,
+ LIST_CONTAINER,
+ PUT_BLOCK,
+ GET_BLOCK,
+ DELETE_BLOCK,
+ LIST_BLOCK,
+ READ_CHUNK,
+ DELETE_CHUNK,
+ WRITE_CHUNK,
+ LIST_CHUNK,
+ COMPACT_CHUNK,
+ PUT_SMALL_FILE,
+ GET_SMALL_FILE,
+ CLOSE_CONTAINER,
+ GET_COMMITTED_BLOCK_LENGTH;
@Override
public String getAction() {
- return this.action;
+ return this.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
index 76cd39a..d2da3e6 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java
@@ -22,30 +22,24 @@ package org.apache.hadoop.ozone.audit;
*/
public enum DummyAction implements AuditAction {
- CREATE_VOLUME("CREATE_VOLUME"),
- CREATE_BUCKET("CREATE_BUCKET"),
- CREATE_KEY("CREATE_KEY"),
- READ_VOLUME("READ_VOLUME"),
- READ_BUCKET("READ_BUCKET"),
- READ_KEY("READ_BUCKET"),
- UPDATE_VOLUME("UPDATE_VOLUME"),
- UPDATE_BUCKET("UPDATE_BUCKET"),
- UPDATE_KEY("UPDATE_KEY"),
- DELETE_VOLUME("DELETE_VOLUME"),
- DELETE_BUCKET("DELETE_BUCKET"),
- DELETE_KEY("DELETE_KEY"),
- SET_OWNER("SET_OWNER"),
- SET_QUOTA("SET_QUOTA");
-
- private final String action;
-
- DummyAction(String action) {
- this.action = action;
- }
+ CREATE_VOLUME,
+ CREATE_BUCKET,
+ CREATE_KEY,
+ READ_VOLUME,
+ READ_BUCKET,
+ READ_KEY,
+ UPDATE_VOLUME,
+ UPDATE_BUCKET,
+ UPDATE_KEY,
+ DELETE_VOLUME,
+ DELETE_BUCKET,
+ DELETE_KEY,
+ SET_OWNER,
+ SET_QUOTA;
@Override
public String getAction() {
- return this.action;
+ return this.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184cced5/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
index 1d4d646..8794014 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
@@ -23,40 +23,34 @@ package org.apache.hadoop.ozone.audit;
public enum OMAction implements AuditAction {
// WRITE Actions
- ALLOCATE_BLOCK("ALLOCATE_BLOCK"),
- ALLOCATE_KEY("ALLOCATE_KEY"),
- COMMIT_KEY("COMMIT_KEY"),
- CREATE_VOLUME("CREATE_VOLUME"),
- CREATE_BUCKET("CREATE_BUCKET"),
- CREATE_KEY("CREATE_KEY"),
- DELETE_VOLUME("DELETE_VOLUME"),
- DELETE_BUCKET("DELETE_BUCKET"),
- DELETE_KEY("DELETE_KEY"),
- RENAME_KEY("RENAME_KEY"),
- SET_OWNER("SET_OWNER"),
- SET_QUOTA("SET_QUOTA"),
- UPDATE_VOLUME("UPDATE_VOLUME"),
- UPDATE_BUCKET("UPDATE_BUCKET"),
- UPDATE_KEY("UPDATE_KEY"),
+ ALLOCATE_BLOCK,
+ ALLOCATE_KEY,
+ COMMIT_KEY,
+ CREATE_VOLUME,
+ CREATE_BUCKET,
+ CREATE_KEY,
+ DELETE_VOLUME,
+ DELETE_BUCKET,
+ DELETE_KEY,
+ RENAME_KEY,
+ SET_OWNER,
+ SET_QUOTA,
+ UPDATE_VOLUME,
+ UPDATE_BUCKET,
+ UPDATE_KEY,
// READ Actions
- CHECK_VOLUME_ACCESS("CHECK_VOLUME_ACCESS"),
- LIST_BUCKETS("LIST_BUCKETS"),
- LIST_VOLUMES("LIST_VOLUMES"),
- LIST_KEYS("LIST_KEYS"),
- READ_VOLUME("READ_VOLUME"),
- READ_BUCKET("READ_BUCKET"),
- READ_KEY("READ_BUCKET"),
- LIST_S3BUCKETS("LIST_S3BUCKETS");
-
- private String action;
-
- OMAction(String action) {
- this.action = action;
- }
+ CHECK_VOLUME_ACCESS,
+ LIST_BUCKETS,
+ LIST_VOLUMES,
+ LIST_KEYS,
+ READ_VOLUME,
+ READ_BUCKET,
+ READ_KEY,
+ LIST_S3BUCKETS;
@Override
public String getAction() {
- return this.action;
+ return this.toString();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-15959. Revert
"HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@'
to be non-simple"
Posted by sh...@apache.org.
HADOOP-15959. Revert "HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@' to be non-simple"
This reverts commit 829a2e4d271f05afb209ddc834cd4a0e85492eda.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0edd372
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0edd372
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0edd372
Branch: refs/heads/HDFS-12943
Commit: d0edd37269bb40290b409d583bcf3b70897c13e0
Parents: 5e102f9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 29 17:52:11 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 29 17:52:11 2018 +0000
----------------------------------------------------------------------
.../authentication/util/KerberosName.java | 9 ++--
.../TestKerberosAuthenticationHandler.java | 7 ++-
.../authentication/util/TestKerberosName.java | 17 ++++++--
.../java/org/apache/hadoop/security/KDiag.java | 46 +-------------------
.../src/site/markdown/SecureMode.md | 6 ---
.../org/apache/hadoop/security/TestKDiag.java | 16 -------
.../security/TestUserGroupInformation.java | 27 ++++--------
7 files changed, 33 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
index 4e7ee3c..287bb13 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
@@ -324,8 +324,8 @@ public class KerberosName {
}
}
if (result != null && nonSimplePattern.matcher(result).find()) {
- LOG.info("Non-simple name {} after auth_to_local rule {}",
- result, this);
+ throw new NoMatchingRule("Non-simple name " + result +
+ " after auth_to_local rule " + this);
}
if (toLowerCase && result != null) {
result = result.toLowerCase(Locale.ENGLISH);
@@ -378,7 +378,7 @@ public class KerberosName {
/**
* Get the translation of the principal name into an operating system
* user name.
- * @return the user name
+ * @return the short name
* @throws IOException throws if something is wrong with the rules
*/
public String getShortName() throws IOException {
@@ -398,8 +398,7 @@ public class KerberosName {
return result;
}
}
- LOG.info("No auth_to_local rules applied to {}", this);
- return toString();
+ throw new NoMatchingRule("No rules applied to " + toString());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
index e672391..8b4bc15 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
@@ -108,7 +108,12 @@ public class TestKerberosAuthenticationHandler
kn = new KerberosName("bar@BAR");
Assert.assertEquals("bar", kn.getShortName());
kn = new KerberosName("bar@FOO");
- Assert.assertEquals("bar@FOO", kn.getShortName());
+ try {
+ kn.getShortName();
+ Assert.fail();
+ }
+ catch (Exception ex) {
+ }
}
@Test(timeout=60000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
index c584fce..2db0df4 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
@@ -72,14 +72,23 @@ public class TestKerberosName {
}
}
+ private void checkBadTranslation(String from) {
+ System.out.println("Checking bad translation for " + from);
+ KerberosName nm = new KerberosName(from);
+ try {
+ nm.getShortName();
+ Assert.fail("didn't get exception for " + from);
+ } catch (IOException ie) {
+ // PASS
+ }
+ }
+
@Test
public void testAntiPatterns() throws Exception {
checkBadName("owen/owen/owen@FOO.COM");
checkBadName("owen@foo/bar.com");
-
- // no rules applied, these should pass
- checkTranslation("foo@ACME.COM", "foo@ACME.COM");
- checkTranslation("root/joe@FOO.COM", "root/joe@FOO.COM");
+ checkBadTranslation("foo@ACME.COM");
+ checkBadTranslation("root/joe@FOO.COM");
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
index b4e535c..1711921 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
@@ -22,7 +22,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ExitUtil;
@@ -55,7 +54,6 @@ import java.util.Collections;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
-import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
import static org.apache.hadoop.security.UserGroupInformation.*;
@@ -131,12 +129,6 @@ public class KDiag extends Configured implements Tool, Closeable {
private boolean nofail = false;
private boolean nologin = false;
private boolean jaas = false;
- private boolean checkShortName = false;
-
- /**
- * A pattern that recognizes simple/non-simple names. Per KerberosName
- */
- private static final Pattern nonSimplePattern = Pattern.compile("[/@]");
/**
* Flag set to true if a {@link #verify(boolean, String, String, Object...)}
@@ -165,8 +157,6 @@ public class KDiag extends Configured implements Tool, Closeable {
public static final String ARG_SECURE = "--secure";
- public static final String ARG_VERIFYSHORTNAME = "--verifyshortname";
-
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public KDiag(Configuration conf,
PrintWriter out,
@@ -210,7 +200,6 @@ public class KDiag extends Configured implements Tool, Closeable {
nofail = popOption(ARG_NOFAIL, args);
jaas = popOption(ARG_JAAS, args);
nologin = popOption(ARG_NOLOGIN, args);
- checkShortName = popOption(ARG_VERIFYSHORTNAME, args);
// look for list of resources
String resource;
@@ -256,9 +245,7 @@ public class KDiag extends Configured implements Tool, Closeable {
+ arg(ARG_NOLOGIN, "", "Do not attempt to log in")
+ arg(ARG_OUTPUT, "<file>", "Write output to a file")
+ arg(ARG_RESOURCE, "<resource>", "Load an XML configuration resource")
- + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure")
- + arg(ARG_VERIFYSHORTNAME, ARG_PRINCIPAL + " <principal>",
- "Verify the short name of the specific principal does not contain '@' or '/'");
+ + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure");
}
private String arg(String name, String params, String meaning) {
@@ -291,7 +278,6 @@ public class KDiag extends Configured implements Tool, Closeable {
println("%s = %d", ARG_KEYLEN, minKeyLength);
println("%s = %s", ARG_KEYTAB, keytab);
println("%s = %s", ARG_PRINCIPAL, principal);
- println("%s = %s", ARG_VERIFYSHORTNAME, checkShortName);
// Fail fast on a JVM without JCE installed.
validateKeyLength();
@@ -391,10 +377,6 @@ public class KDiag extends Configured implements Tool, Closeable {
validateJAAS(jaas);
validateNTPConf();
- if (checkShortName) {
- validateShortName();
- }
-
if (!nologin) {
title("Logging in");
if (keytab != null) {
@@ -449,32 +431,6 @@ public class KDiag extends Configured implements Tool, Closeable {
}
/**
- * Verify whether auth_to_local rules transform a principal name
- * <p>
- * Having a local user name "bar@foo.com" may be harmless, so it is noted at
- * info. However if what was intended is a transformation to "bar"
- * it can be difficult to debug, hence this check.
- */
- protected void validateShortName() {
- failif(principal == null, CAT_KERBEROS, "No principal defined");
-
- try {
- KerberosName kn = new KerberosName(principal);
- String result = kn.getShortName();
- if (nonSimplePattern.matcher(result).find()) {
- warn(CAT_KERBEROS, principal + " short name: " + result
- + " still contains @ or /");
- }
- } catch (IOException e) {
- throw new KerberosDiagsFailure(CAT_KERBEROS, e,
- "Failed to get short name for " + principal, e);
- } catch (IllegalArgumentException e) {
- error(CAT_KERBEROS, "KerberosName(" + principal + ") failed: %s\n%s",
- e, StringUtils.stringifyException(e));
- }
- }
-
- /**
* Get the default realm.
* <p>
* Not having a default realm may be harmless, so is noted at info.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index ecf94eb..09f1eed 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -470,7 +470,6 @@ KDiag: Diagnose Kerberos Problems
[--out <file>] : Write output to a file.
[--resource <resource>] : Load an XML configuration resource.
[--secure] : Require the hadoop configuration to be secure.
- [--verifyshortname <principal>]: Verify the short name of the specific principal does not contain '@' or '/'
```
#### `--jaas`: Require a JAAS file to be defined in `java.security.auth.login.config`.
@@ -566,11 +565,6 @@ or implicitly set to "simple":
Needless to say, an application so configured cannot talk to a secure Hadoop cluster.
-#### `--verifyshortname <principal>`: validate the short name of a principal
-
-This verifies that the short name of a principal contains neither the `"@"`
-nor `"/"` characters.
-
### Example
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
index e395566..3895ae1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
@@ -165,22 +165,6 @@ public class TestKDiag extends Assert {
}
@Test
- public void testKerberosName() throws Throwable {
- kdiagFailure(ARG_KEYLEN, KEYLEN,
- ARG_VERIFYSHORTNAME,
- ARG_PRINCIPAL, "foo/foo/foo@BAR.COM");
- }
-
- @Test
- public void testShortName() throws Throwable {
- kdiag(ARG_KEYLEN, KEYLEN,
- ARG_KEYTAB, keytab.getAbsolutePath(),
- ARG_PRINCIPAL,
- ARG_VERIFYSHORTNAME,
- ARG_PRINCIPAL, "foo@EXAMPLE.COM");
- }
-
- @Test
public void testFileOutput() throws Throwable {
File f = new File("target/kdiag.txt");
kdiag(ARG_KEYLEN, KEYLEN,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0edd372/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 3020f9b..8af0328 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -332,15 +332,10 @@ public class TestUserGroupInformation {
UserGroupInformation.setConfiguration(conf);
testConstructorSuccess("user1", "user1");
testConstructorSuccess("user4@OTHER.REALM", "other-user4");
-
- // pass through test, no transformation
- testConstructorSuccess("user2@DEFAULT.REALM", "user2@DEFAULT.REALM");
- testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3/cron@DEFAULT.REALM");
- testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
- // failures
- testConstructorFailures("user6@example.com@OTHER.REALM");
- testConstructorFailures("user7@example.com@DEFAULT.REALM");
+ // failure test
+ testConstructorFailures("user2@DEFAULT.REALM");
+ testConstructorFailures("user3/cron@DEFAULT.REALM");
+ testConstructorFailures("user5/cron@OTHER.REALM");
testConstructorFailures(null);
testConstructorFailures("");
}
@@ -354,13 +349,10 @@ public class TestUserGroupInformation {
testConstructorSuccess("user1", "user1");
testConstructorSuccess("user2@DEFAULT.REALM", "user2");
- testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");
-
- // no rules applied, local name remains the same
- testConstructorSuccess("user4@OTHER.REALM", "user4@OTHER.REALM");
- testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
+ testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");
// failure test
+ testConstructorFailures("user4@OTHER.REALM");
+ testConstructorFailures("user5/cron@OTHER.REALM");
testConstructorFailures(null);
testConstructorFailures("");
}
@@ -401,9 +393,8 @@ public class TestUserGroupInformation {
} catch (IllegalArgumentException e) {
String expect = (userName == null || userName.isEmpty())
? "Null user" : "Illegal principal name "+userName;
- String expect2 = "Malformed Kerberos name: "+userName;
- assertTrue("Did not find "+ expect + " or " + expect2 + " in " + e,
- e.toString().contains(expect) || e.toString().contains(expect2));
+ assertTrue("Did not find "+ expect + " in " + e,
+ e.toString().contains(expect));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-8989. [YARN-8851] Move
DockerCommandPlugin volume related APIs' invocation from
DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun
Tang via wangda)
Posted by sh...@apache.org.
YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun Tang via wangda)
Change-Id: Ia6d532c687168448416dfdf46f0ac34bff20e6ca
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe7dab8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe7dab8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe7dab8e
Branch: refs/heads/HDFS-12943
Commit: fe7dab8ef55f08cf18c2d62c782c1ab8930a5a15
Parents: 8976439
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:55:16 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 15:03:06 2018 -0800
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 44 ++++++++------------
.../runtime/TestDockerContainerRuntime.java | 15 ++++---
2 files changed, 24 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 15ff0ff..225bc19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -456,32 +456,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
-
- // Create volumes when needed.
- if (nmContext != null
- && nmContext.getResourcePluginManager().getNameToPlugins() != null) {
- for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
- .getNameToPlugins().values()) {
- DockerCommandPlugin dockerCommandPlugin =
- plugin.getDockerCommandPluginInstance();
- if (dockerCommandPlugin != null) {
- DockerVolumeCommand dockerVolumeCommand =
- dockerCommandPlugin.getCreateDockerVolumeCommand(
- ctx.getContainer());
- if (dockerVolumeCommand != null) {
- runDockerVolumeCommand(dockerVolumeCommand, container);
-
- // After volume created, run inspect to make sure volume properly
- // created.
- if (dockerVolumeCommand.getSubCommand().equals(
- DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
- checkDockerVolumeCreated(dockerVolumeCommand, container);
- }
- }
- }
- }
- }
}
private void checkDockerVolumeCreated(
@@ -1034,14 +1008,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
- // use plugins to update docker run command.
+ // use plugins to create volume and update docker run command.
if (nmContext != null
&& nmContext.getResourcePluginManager().getNameToPlugins() != null) {
for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
.getNameToPlugins().values()) {
DockerCommandPlugin dockerCommandPlugin =
plugin.getDockerCommandPluginInstance();
+
if (dockerCommandPlugin != null) {
+ // Create volumes when needed.
+ DockerVolumeCommand dockerVolumeCommand =
+ dockerCommandPlugin.getCreateDockerVolumeCommand(
+ ctx.getContainer());
+ if (dockerVolumeCommand != null) {
+ runDockerVolumeCommand(dockerVolumeCommand, container);
+
+ // After volume created, run inspect to make sure volume properly
+ // created.
+ if (dockerVolumeCommand.getSubCommand().equals(
+ DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
+ checkDockerVolumeCreated(dockerVolumeCommand, container);
+ }
+ }
+ // Update cmd
dockerCommandPlugin.updateDockerRunCommand(runCommand, container);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 40ac618..0b5ce3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -1958,16 +1958,16 @@ public class TestDockerContainerRuntime {
ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
PrivilegedOperation.class);
- //single invocation expected
+ //Three invocations expected (volume creation, volume check, run container)
//due to type erasure + mocking, this verification requires a suppress
// warning annotation on the entire method
- verify(mockExecutor, times(2))
+ verify(mockExecutor, times(3))
.executePrivilegedOperation(anyList(), opCaptor.capture(), any(
File.class), anyMap(), anyBoolean(), anyBoolean());
//verification completed. we need to isolate specific invications.
// hence, reset mock here
- Mockito.reset(mockExecutor);
+ //Mockito.reset(mockExecutor);
List<PrivilegedOperation> allCaptures = opCaptor.getAllValues();
@@ -2070,10 +2070,8 @@ public class TestDockerContainerRuntime {
try {
runtime.prepareContainer(containerRuntimeContext);
-
- checkVolumeCreateCommand();
-
runtime.launchContainer(containerRuntimeContext);
+ checkVolumeCreateCommand();
} catch (ContainerExecutionException e) {
if (expectFail) {
// Expected
@@ -2166,10 +2164,11 @@ public class TestDockerContainerRuntime {
ContainerRuntimeContext containerRuntimeContext = builder.build();
runtime.prepareContainer(containerRuntimeContext);
- checkVolumeCreateCommand();
runtime.launchContainer(containerRuntimeContext);
- List<String> dockerCommands = readDockerCommands();
+ checkVolumeCreateCommand();
+
+ List<String> dockerCommands = readDockerCommands(3);
int expected = 14;
int counter = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDDS-882. Provide a config to
optionally turn on/off the sync flag during chunk writes. Contributed by
Shashikant Banerjee.
Posted by sh...@apache.org.
HDDS-882. Provide a config to optionally turn on/off the sync flag during chunk writes. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f3e12ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f3e12ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f3e12ff
Branch: refs/heads/HDFS-12943
Commit: 8f3e12ff07f5a8490af23f3ca231f97b381682e5
Parents: 5a3c771
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Sun Dec 2 08:06:24 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Sun Dec 2 08:06:24 2018 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 +++
hadoop-hdds/common/src/main/resources/ozone-default.xml | 8 ++++++++
.../hadoop/ozone/container/keyvalue/KeyValueHandler.java | 7 ++++++-
.../ozone/container/keyvalue/helpers/ChunkUtils.java | 11 ++++++++---
.../ozone/container/keyvalue/impl/ChunkManagerImpl.java | 10 ++++++++--
.../ozone/container/keyvalue/TestChunkManagerImpl.java | 2 +-
.../container/common/impl/TestContainerPersistence.java | 2 +-
7 files changed, 35 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3e12ff/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index df233f7..496861c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -52,6 +52,9 @@ public final class OzoneConfigKeys {
public static final boolean DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT =
false;
+ public static final String DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY =
+ "dfs.container.chunk.write.sync";
+ public static final boolean DFS_CONTAINER_CHUNK_WRITE_SYNC_DEFAULT = true;
/**
* Ratis Port where containers listen to.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3e12ff/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 0545805..edce616 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -53,6 +53,14 @@
</description>
</property>
<property>
+ <name>dfs.container.chunk.write.sync</name>
+ <value>true</value>
+ <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+ <description>Determines whether the chunk writes in the container happen as
+ sync I/0 or buffered I/O operation.
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.statemachinedata.sync.timeout</name>
<value>10s</value>
<tag>OZONE, DEBUG, CONTAINER, RATIS</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3e12ff/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index b4cfcd0..5130253 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
@@ -111,13 +112,17 @@ public class KeyValueHandler extends Handler {
private final VolumeChoosingPolicy volumeChoosingPolicy;
private final long maxContainerSize;
private final AutoCloseableLock handlerLock;
+ private final boolean doSyncWrite;
public KeyValueHandler(Configuration config, StateContext context,
ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics) {
super(config, context, contSet, volSet, metrics);
containerType = ContainerType.KeyValueContainer;
blockManager = new BlockManagerImpl(config);
- chunkManager = new ChunkManagerImpl();
+ doSyncWrite =
+ conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY,
+ OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_DEFAULT);
+ chunkManager = new ChunkManagerImpl(doSyncWrite);
long svcInterval = config
.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3e12ff/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
index 7239843..6df6f3d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -67,10 +67,11 @@ public final class ChunkUtils {
* @param chunkInfo - Data stream to write.
* @param data - The data buffer.
* @param volumeIOStats
+ * @param sync whether to do fsync or not
* @throws StorageContainerException
*/
public static void writeData(File chunkFile, ChunkInfo chunkInfo,
- ByteBuffer data, VolumeIOStats volumeIOStats)
+ ByteBuffer data, VolumeIOStats volumeIOStats, boolean sync)
throws StorageContainerException, ExecutionException,
InterruptedException, NoSuchAlgorithmException {
int bufferSize = data.capacity();
@@ -88,12 +89,16 @@ public final class ChunkUtils {
try {
long writeTimeStart = Time.monotonicNow();
- file =
+ file = sync ?
AsynchronousFileChannel.open(chunkFile.toPath(),
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.SPARSE,
- StandardOpenOption.SYNC);
+ StandardOpenOption.SYNC) :
+ AsynchronousFileChannel.open(chunkFile.toPath(),
+ StandardOpenOption.CREATE,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.SPARSE);
lock = file.lock().get();
int size = file.write(data, chunkInfo.getOffset()).get();
// Increment volumeIO stats here.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3e12ff/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index 38430af..a2e8e5c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -54,6 +54,11 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Res
*/
public class ChunkManagerImpl implements ChunkManager {
static final Logger LOG = LoggerFactory.getLogger(ChunkManagerImpl.class);
+ private final boolean doSyncWrite;
+
+ public ChunkManagerImpl(boolean sync) {
+ doSyncWrite = sync;
+ }
/**
* writes a given chunk.
@@ -115,7 +120,8 @@ public class ChunkManagerImpl implements ChunkManager {
"tmpChunkFile already exists" + tmpChunkFile + "Overwriting it.");
}
// Initially writes to temporary chunk file.
- ChunkUtils.writeData(tmpChunkFile, info, data, volumeIOStats);
+ ChunkUtils
+ .writeData(tmpChunkFile, info, data, volumeIOStats, doSyncWrite);
// No need to increment container stats here, as still data is not
// committed here.
break;
@@ -139,7 +145,7 @@ public class ChunkManagerImpl implements ChunkManager {
break;
case COMBINED:
// directly write to the chunk file
- ChunkUtils.writeData(chunkFile, info, data, volumeIOStats);
+ ChunkUtils.writeData(chunkFile, info, data, volumeIOStats, doSyncWrite);
if (!isOverwrite) {
containerData.incrBytesUsed(info.getLen());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3e12ff/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index f17288f..fd48bf5 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -99,7 +99,7 @@ public class TestChunkManagerImpl {
.getLocalID(), 0), 0, data.length);
// Create a ChunkManager object.
- chunkManager = new ChunkManagerImpl();
+ chunkManager = new ChunkManagerImpl(true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f3e12ff/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 1637ff7..32b01ae 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -131,7 +131,7 @@ public class TestContainerPersistence {
containerSet = new ContainerSet();
volumeSet = new VolumeSet(DATANODE_UUID, conf);
blockManager = new BlockManagerImpl(conf);
- chunkManager = new ChunkManagerImpl();
+ chunkManager = new ChunkManagerImpl(true);
for (String dir : conf.getStrings(ScmConfigKeys.HDDS_DATANODE_DIR_KEY)) {
StorageLocation location = StorageLocation.parse(dir);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: Revert "HADOOP-15852. Refactor
QuotaUsage. Contributed by Beluga Behr."
Posted by sh...@apache.org.
Revert "HADOOP-15852. Refactor QuotaUsage. Contributed by Beluga Behr."
This reverts commit fb9deed41d6b9f242474b474a5acde0c858e28f6.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa89492f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa89492f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa89492f
Branch: refs/heads/HDFS-12943
Commit: aa89492f29fafa471f99ff225602752bdb9b5c8f
Parents: e89941f
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Tue Dec 4 12:57:28 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Tue Dec 4 12:57:28 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/QuotaUsage.java | 108 +++++++++----------
1 file changed, 53 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa89492f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
index 4e42e5b..3472362 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -40,12 +40,14 @@ public class QuotaUsage {
/** Builder class for QuotaUsage. */
public static class Builder {
public Builder() {
- this.quota = -1L;
- this.spaceQuota = -1L;
+ this.quota = -1;
+ this.spaceQuota = -1;
typeConsumed = new long[StorageType.values().length];
typeQuota = new long[StorageType.values().length];
- Arrays.fill(typeQuota, -1L);
+ for (int i = 0; i < typeQuota.length; i++) {
+ typeQuota[i] = -1;
+ }
}
public Builder fileAndDirectoryCount(long count) {
@@ -69,8 +71,9 @@ public class QuotaUsage {
}
public Builder typeConsumed(long[] typeConsumed) {
- System.arraycopy(typeConsumed, 0, this.typeConsumed, 0,
- typeConsumed.length);
+ for (int i = 0; i < typeConsumed.length; i++) {
+ this.typeConsumed[i] = typeConsumed[i];
+ }
return this;
}
@@ -85,8 +88,9 @@ public class QuotaUsage {
}
public Builder typeQuota(long[] typeQuota) {
- System.arraycopy(typeQuota, 0, this.typeQuota, 0,
- typeQuota.length);
+ for (int i = 0; i < typeQuota.length; i++) {
+ this.typeQuota[i] = typeQuota[i];
+ }
return this;
}
@@ -149,12 +153,22 @@ public class QuotaUsage {
/** Return storage type quota. */
public long getTypeQuota(StorageType type) {
- return (typeQuota != null) ? typeQuota[type.ordinal()] : -1L;
+ return (typeQuota != null) ? typeQuota[type.ordinal()] : -1;
}
/** Return storage type consumed. */
public long getTypeConsumed(StorageType type) {
- return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0L;
+ return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0;
+ }
+
+ /** Return storage type quota. */
+ private long[] getTypesQuota() {
+ return typeQuota;
+ }
+
+ /** Return storage type quota. */
+ private long[] getTypesConsumed() {
+ return typeConsumed;
}
/** Return true if any storage type quota has been set. */
@@ -163,7 +177,7 @@ public class QuotaUsage {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeQuota[t.ordinal()] > 0L) {
+ if (typeQuota[t.ordinal()] > 0) {
return true;
}
}
@@ -176,7 +190,7 @@ public class QuotaUsage {
return false;
}
for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeConsumed[t.ordinal()] > 0L) {
+ if (typeConsumed[t.ordinal()] > 0) {
return true;
}
}
@@ -184,50 +198,33 @@ public class QuotaUsage {
}
@Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result
- + (int) (fileAndDirectoryCount ^ (fileAndDirectoryCount >>> 32));
- result = prime * result + (int) (quota ^ (quota >>> 32));
- result = prime * result + (int) (spaceConsumed ^ (spaceConsumed >>> 32));
- result = prime * result + (int) (spaceQuota ^ (spaceQuota >>> 32));
- result = prime * result + Arrays.hashCode(typeConsumed);
- result = prime * result + Arrays.hashCode(typeQuota);
- return result;
+ public boolean equals(Object to) {
+ return (this == to || (to instanceof QuotaUsage &&
+ getFileAndDirectoryCount() ==
+ ((QuotaUsage) to).getFileAndDirectoryCount() &&
+ getQuota() == ((QuotaUsage) to).getQuota() &&
+ getSpaceConsumed() == ((QuotaUsage) to).getSpaceConsumed() &&
+ getSpaceQuota() == ((QuotaUsage) to).getSpaceQuota() &&
+ Arrays.equals(getTypesQuota(), ((QuotaUsage) to).getTypesQuota()) &&
+ Arrays.equals(getTypesConsumed(),
+ ((QuotaUsage) to).getTypesConsumed())));
}
@Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- QuotaUsage other = (QuotaUsage) obj;
- if (fileAndDirectoryCount != other.fileAndDirectoryCount) {
- return false;
- }
- if (quota != other.quota) {
- return false;
- }
- if (spaceConsumed != other.spaceConsumed) {
- return false;
- }
- if (spaceQuota != other.spaceQuota) {
- return false;
- }
- if (!Arrays.equals(typeConsumed, other.typeConsumed)) {
- return false;
+ public int hashCode() {
+ long result = (getFileAndDirectoryCount() ^ getQuota() ^
+ getSpaceConsumed() ^ getSpaceQuota());
+ if (getTypesQuota() != null) {
+ for (long quota : getTypesQuota()) {
+ result ^= quota;
+ }
}
- if (!Arrays.equals(typeQuota, other.typeQuota)) {
- return false;
+ if (getTypesConsumed() != null) {
+ for (long consumed : getTypesConsumed()) {
+ result ^= consumed;
+ }
}
- return true;
+ return (int)result;
}
/**
@@ -295,11 +292,11 @@ public class QuotaUsage {
String spaceQuotaStr = QUOTA_NONE;
String spaceQuotaRem = QUOTA_INF;
- if (quota > 0L) {
+ if (quota > 0) {
quotaStr = formatSize(quota, hOption);
quotaRem = formatSize(quota-fileAndDirectoryCount, hOption);
}
- if (spaceQuota >= 0L) {
+ if (spaceQuota >= 0) {
spaceQuotaStr = formatSize(spaceQuota, hOption);
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
@@ -310,7 +307,7 @@ public class QuotaUsage {
protected String getTypesQuotaUsage(boolean hOption,
List<StorageType> types) {
- StringBuilder content = new StringBuilder();
+ StringBuffer content = new StringBuffer();
for (StorageType st : types) {
long typeQuota = getTypeQuota(st);
long typeConsumed = getTypeConsumed(st);
@@ -335,12 +332,13 @@ public class QuotaUsage {
* @return storage header string
*/
public static String getStorageTypeHeader(List<StorageType> storageTypes) {
- StringBuilder header = new StringBuilder();
+ StringBuffer header = new StringBuffer();
+
for (StorageType st : storageTypes) {
/* the field length is 13/17 for quota and remain quota
* as the max length for quota name is ARCHIVE_QUOTA
* and remain quota name REM_ARCHIVE_QUOTA */
- final String storageName = st.toString();
+ String storageName = st.toString();
header.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT,
storageName + "_QUOTA", "REM_" + storageName + "_QUOTA"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-9067. YARN Resource Manager
is running OOM because of leak of Configuration Object. Contributed by Eric
Yang.
Posted by sh...@apache.org.
YARN-9067. YARN Resource Manager is running OOM because of leak of Configuration Object. Contributed by Eric Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efc4d91c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efc4d91c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efc4d91c
Branch: refs/heads/HDFS-12943
Commit: efc4d91cbeab8a13f6d61cb0e56443adb2d77559
Parents: fe7dab8
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Nov 29 09:34:14 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Nov 29 09:34:14 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/service/webapp/ApiServer.java | 209 +++++++++++--------
.../hadoop/yarn/service/ServiceClientTest.java | 2 +-
.../yarn/service/client/ServiceClient.java | 1 +
3 files changed, 126 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index db831ba..88aeefd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -118,10 +118,13 @@ public class ApiServer {
@Override
public Void run() throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- sc.actionBuild(service);
- sc.close();
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionBuild(service);
+ } finally {
+ sc.close();
+ }
return null;
}
});
@@ -133,11 +136,14 @@ public class ApiServer {
@Override
public ApplicationId run() throws IOException, YarnException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- ApplicationId applicationId = sc.actionCreate(service);
- sc.close();
- return applicationId;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId applicationId = sc.actionCreate(service);
+ return applicationId;
+ } finally {
+ sc.close();
+ }
}
});
serviceStatus.setDiagnostics("Application ID: " + applicationId);
@@ -245,29 +251,32 @@ public class ApiServer {
public Integer run() throws Exception {
int result = 0;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Exception stopException = null;
try {
- result = sc.actionStop(appName, destroy);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully stopped service {}", appName);
- }
- } catch (Exception e) {
- LOG.info("Got exception stopping service", e);
- stopException = e;
- }
- if (destroy) {
- result = sc.actionDestroy(appName);
- if (result == EXIT_SUCCESS) {
- LOG.info("Successfully deleted service {}", appName);
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Exception stopException = null;
+ try {
+ result = sc.actionStop(appName, destroy);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully stopped service {}", appName);
+ }
+ } catch (Exception e) {
+ LOG.info("Got exception stopping service", e);
+ stopException = e;
}
- } else {
- if (stopException != null) {
- throw stopException;
+ if (destroy) {
+ result = sc.actionDestroy(appName);
+ if (result == EXIT_SUCCESS) {
+ LOG.info("Successfully deleted service {}", appName);
+ }
+ } else {
+ if (stopException != null) {
+ throw stopException;
+ }
}
+ } finally {
+ sc.close();
}
- sc.close();
return result;
}
});
@@ -378,13 +387,16 @@ public class ApiServer {
@Override
public Map<String, Long> run() throws YarnException, IOException {
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Map<String, Long> original = sc.flexByRestService(appName,
- Collections.singletonMap(componentName,
- component.getNumberOfContainers()));
- sc.close();
- return original;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Map<String, Long> original = sc.flexByRestService(appName,
+ Collections.singletonMap(componentName,
+ component.getNumberOfContainers()));
+ return original;
+ } finally {
+ sc.close();
+ }
}
});
ServiceStatus status = new ServiceStatus();
@@ -632,12 +644,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException {
int result = 0;
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc
- .actionFlex(appName, componentCountStrings);
- sc.close();
- return Integer.valueOf(result);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionFlex(appName, componentCountStrings);
+ return Integer.valueOf(result);
+ } finally {
+ sc.close();
+ }
}
});
if (result == EXIT_SUCCESS) {
@@ -658,12 +673,15 @@ public class ApiServer {
@Override
public String run() throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- String newLifeTime = sc.updateLifetime(appName,
- updateAppData.getLifetime());
- sc.close();
- return newLifeTime;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ String newLifeTime = sc.updateLifetime(appName,
+ updateAppData.getLifetime());
+ return newLifeTime;
+ } finally {
+ sc.close();
+ }
}
});
ServiceStatus status = new ServiceStatus();
@@ -681,11 +699,14 @@ public class ApiServer {
@Override public ApplicationId run()
throws YarnException, IOException {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- ApplicationId appId = sc.actionStartAndGetId(appName);
- sc.close();
- return appId;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ ApplicationId appId = sc.actionStartAndGetId(appName);
+ return appId;
+ } finally {
+ sc.close();
+ }
}
});
LOG.info("Successfully started service " + appName);
@@ -702,14 +723,17 @@ public class ApiServer {
ServiceStatus status = new ServiceStatus();
ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
- sc.actionUpgradeExpress(service);
- } else {
- sc.initiateUpgrade(service);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
+ sc.actionUpgradeExpress(service);
+ } else {
+ sc.initiateUpgrade(service);
+ }
+ } finally {
+ sc.close();
}
- sc.close();
return null;
});
LOG.info("Service {} version {} upgrade initialized", service.getName(),
@@ -724,11 +748,14 @@ public class ApiServer {
final UserGroupInformation ugi) throws IOException, InterruptedException {
int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- int exitCode = sc.actionCancelUpgrade(serviceName);
- sc.close();
- return exitCode;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ int exitCode = sc.actionCancelUpgrade(serviceName);
+ return exitCode;
+ } finally {
+ sc.close();
+ }
});
if (result == EXIT_SUCCESS) {
ServiceStatus status = new ServiceStatus();
@@ -793,10 +820,13 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
int result1;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result1 = sc.actionUpgrade(service, containers);
- sc.close();
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result1 = sc.actionUpgrade(service, containers);
+ } finally {
+ sc.close();
+ }
return result1;
});
}
@@ -815,12 +845,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException {
int result = 0;
ServiceClient sc = new ServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc
- .actionDecommissionInstances(appName, instances);
- sc.close();
- return Integer.valueOf(result);
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc
+ .actionDecommissionInstances(appName, instances);
+ return Integer.valueOf(result);
+ } finally {
+ sc.close();
+ }
}
});
if (result == EXIT_SUCCESS) {
@@ -840,11 +873,14 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Service>) () -> {
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- Service app1 = sc.getStatus(serviceName);
- sc.close();
- return app1;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ Service app1 = sc.getStatus(serviceName);
+ return app1;
+ } finally {
+ sc.close();
+ }
});
}
@@ -855,12 +891,15 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<ComponentContainers[]>) () -> {
ComponentContainers[] result;
ServiceClient sc = getServiceClient();
- sc.init(YARN_CONFIG);
- sc.start();
- result = sc.getContainers(serviceName, componentNames, version,
- containerStates);
- sc.close();
- return result;
+ try {
+ sc.init(YARN_CONFIG);
+ sc.start();
+ result = sc.getContainers(serviceName, componentNames, version,
+ containerStates);
+ return result;
+ } finally {
+ sc.close();
+ }
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
index d022614..89366b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -75,7 +75,7 @@ public class ServiceClientTest extends ServiceClient {
public void forceStop() {
expectedInstances.clear();
- super.stop();
+ stop();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efc4d91c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 713d890..5cdb8c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -163,6 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
if (registryClient != null) {
registryClient.stop();
}
+ fs.getFileSystem().close();
super.serviceStop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDFS-14112. Avoid recursive call
to external authorizer for getContentSummary.
Posted by sh...@apache.org.
HDFS-14112. Avoid recursive call to external authorizer for getContentSummary.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0081b02e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0081b02e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0081b02e
Branch: refs/heads/HDFS-12943
Commit: 0081b02e35306cb757c63d0f11a536941d73a139
Parents: ae5fbdd
Author: Tsz Wo Nicholas Sze <sz...@apache.org>
Authored: Thu Nov 29 13:55:21 2018 -0800
Committer: Tsz Wo Nicholas Sze <sz...@apache.org>
Committed: Thu Nov 29 13:55:21 2018 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++
.../hdfs/server/namenode/FSDirStatAndListingOp.java | 5 +++++
.../apache/hadoop/hdfs/server/namenode/FSDirectory.java | 7 +++++++
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 10 ++++++++++
4 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0081b02e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 3628b2b..5899c92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -284,6 +284,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT;
public static final String DFS_PERMISSIONS_ENABLED_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_PERMISSIONS_ENABLED_KEY;
+ public static final String DFS_PERMISSIONS_CONTENT_SUMMARY_SUBACCESS_KEY
+ = "dfs.permissions.ContentSummary.subAccess";
+ public static final boolean DFS_PERMISSIONS_CONTENT_SUMMARY_SUBACCESS_DEFAULT
+ = false;
public static final boolean DFS_PERMISSIONS_ENABLED_DEFAULT = true;
public static final String DFS_PERMISSIONS_SUPERUSERGROUP_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0081b02e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 01de236..052e522 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -128,6 +128,11 @@ class FSDirStatAndListingOp {
static ContentSummary getContentSummary(
FSDirectory fsd, FSPermissionChecker pc, String src) throws IOException {
final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.READ_LINK);
+ if (fsd.isPermissionEnabled() && fsd.isPermissionContentSummarySubAccess()) {
+ fsd.checkPermission(pc, iip, false, null, null, null,
+ FsAction.READ_EXECUTE);
+ pc = null;
+ }
// getContentSummaryInt() call will check access (if enabled) when
// traversing all sub directories.
return getContentSummaryInt(fsd, pc, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0081b02e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 712a327..45f859c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -175,6 +175,7 @@ public class FSDirectory implements Closeable {
private final ReentrantReadWriteLock dirLock;
private final boolean isPermissionEnabled;
+ private final boolean isPermissionContentSummarySubAccess;
/**
* Support for ACLs is controlled by a configuration flag. If the
* configuration flag is false, then the NameNode will reject all
@@ -274,6 +275,9 @@ public class FSDirectory implements Closeable {
this.isPermissionEnabled = conf.getBoolean(
DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,
DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT);
+ this.isPermissionContentSummarySubAccess = conf.getBoolean(
+ DFSConfigKeys.DFS_PERMISSIONS_CONTENT_SUMMARY_SUBACCESS_KEY,
+ DFSConfigKeys.DFS_PERMISSIONS_CONTENT_SUMMARY_SUBACCESS_DEFAULT);
this.fsOwnerShortUserName =
UserGroupInformation.getCurrentUser().getShortUserName();
this.supergroup = conf.get(
@@ -538,6 +542,9 @@ public class FSDirectory implements Closeable {
boolean isAclsEnabled() {
return aclsEnabled;
}
+ boolean isPermissionContentSummarySubAccess() {
+ return isPermissionContentSummarySubAccess;
+ }
@VisibleForTesting
public boolean isPosixAclInheritanceEnabled() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0081b02e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index c187a7d..4e6c4db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -470,6 +470,16 @@
</property>
<property>
+ <name>dfs.permissions.ContentSummary.subAccess</name>
+ <value>false</value>
+ <description>
+ If "true", the ContentSummary permission checking will use subAccess.
+ If "false", the ContentSummary permission checking will NOT use subAccess.
+ subAccess means using recursion to check the access of all descendants.
+ </description>
+</property>
+
+<property>
<name>dfs.permissions.superusergroup</name>
<value>supergroup</value>
<description>The name of the group of super-users.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-9057. Removed third party
class bundle from CSI jar file. Contributed by Weiwei Yang
Posted by sh...@apache.org.
YARN-9057. Removed third party class bundle from CSI jar file.
Contributed by Weiwei Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1dabb31c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1dabb31c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1dabb31c
Branch: refs/heads/HDFS-12943
Commit: 1dabb31cdf907cbee418c469368a59393fd52844
Parents: 228156c
Author: Eric Yang <ey...@apache.org>
Authored: Wed Dec 5 15:56:10 2018 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Dec 5 15:56:10 2018 -0500
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 4 +
.../hadoop-yarn/hadoop-yarn-csi/pom.xml | 78 ++++----------------
2 files changed, 19 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1dabb31c/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 4055acb..a5c3c0e 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -220,6 +220,10 @@
<directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/target/lib</directory>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory>
</fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-csi/target/lib</directory>
+ <outputDirectory>share/hadoop/${hadoop.component}/csi/lib</outputDirectory>
+ </fileSet>
</fileSets>
<moduleSets>
<moduleSet>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1dabb31c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
index 27d8452..1a19f0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
@@ -86,14 +86,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
@@ -114,6 +117,18 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeScope>runtime</includeScope>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -148,69 +163,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>${maven-shade-plugin.version}</version>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <relocations>
- <relocation>
- <pattern>io.grpc</pattern>
- <shadedPattern>csi.io.grpc</shadedPattern>
- </relocation>
- <relocation>
- <!-- this includes protobuf-java and guava libraries -->
- <pattern>com.google</pattern>
- <shadedPattern>csi.com.google</shadedPattern>
- </relocation>
- <relocation>
- <pattern>io.netty</pattern>
- <shadedPattern>csi.io.netty</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <!-- We need to rename the native library file in order to let shaded classes to load them -->
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>unpack</id>
- <phase>package</phase>
- <configuration>
- <target>
- <echo message="Unpack hadoop-yarn-csi jar file"/>
- <unzip src="${project.build.directory}/${project.artifactId}-${project.version}.jar"
- dest="${project.build.directory}/unpacked/"/>
- <echo message="Append the shaded prefix to netty's native file in META-INF"/>
- <move file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_epoll_x86_64.so"
- tofile="${project.build.directory}/unpacked/META-INF/native/libcsi_netty_transport_native_epoll_x86_64.so" />
- <echo message="Re-pack the jar"/>
- <jar destfile="${project.build.directory}/${project.artifactId}-${project.version}.jar"
- basedir="${project.build.directory}/unpacked"/>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-14927. ITestS3GuardTool
failures in testDestroyNoBucket(). Contributed by Gabor Bota.
Posted by sh...@apache.org.
HADOOP-14927. ITestS3GuardTool failures in testDestroyNoBucket(). Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb0d3a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb0d3a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb0d3a3
Branch: refs/heads/HDFS-12943
Commit: 7eb0d3a32435da110dc9e6004dba8c5c9b082c35
Parents: 184cced
Author: Sean Mackrory <ma...@apache.org>
Authored: Wed Nov 28 16:57:12 2018 -0700
Committer: Sean Mackrory <ma...@apache.org>
Committed: Thu Nov 29 09:36:39 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 38 ++++++++++++--------
1 file changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb0d3a3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 1316121..aea57a6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -218,6 +218,27 @@ public abstract class S3GuardTool extends Configured implements Tool {
format.addOptionWithValue(SECONDS_FLAG);
}
+ protected void checkMetadataStoreUri(List<String> paths) throws IOException {
+ // be sure that path is provided in params, so there's no IOoBE
+ String s3Path = "";
+ if(!paths.isEmpty()) {
+ s3Path = paths.get(0);
+ }
+
+ // Check if DynamoDB url is set from arguments.
+ String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
+ if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
+ // If not set, check if filesystem is guarded by creating an
+ // S3AFileSystem and check if hasMetadataStore is true
+ try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
+ S3AFileSystem.newInstance(toUri(s3Path), getConf())){
+ Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
+ "The S3 bucket is unguarded. " + getName()
+ + " can not be used on an unguarded bucket.");
+ }
+ }
+ }
+
/**
* Parse metadata store from command line option or HDFS configuration.
*
@@ -500,20 +521,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
Map<String, String> options = new HashMap<>();
- String s3Path = paths.get(0);
-
- // Check if DynamoDB url is set from arguments.
- String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
- if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
- // If not set, check if filesystem is guarded by creating an
- // S3AFileSystem and check if hasMetadataStore is true
- try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
- S3AFileSystem.newInstance(toUri(s3Path), getConf())){
- Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
- "The S3 bucket is unguarded. " + getName()
- + " can not be used on an unguarded bucket.");
- }
- }
+ checkMetadataStoreUri(paths);
String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (StringUtils.isNotEmpty(readCap)) {
@@ -590,6 +598,8 @@ public abstract class S3GuardTool extends Configured implements Tool {
throw e;
}
+ checkMetadataStoreUri(paths);
+
try {
initMetadataStore(false);
} catch (FileNotFoundException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDFS-12946. Add a tool to check
rack configuration against EC policies. Contributed by Kitti Nanasi.
Posted by sh...@apache.org.
HDFS-12946. Add a tool to check rack configuration against EC policies. Contributed by Kitti Nanasi.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd5e7c6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd5e7c6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd5e7c6b
Branch: refs/heads/HDFS-12943
Commit: dd5e7c6b7239a93f2391beaa11181e442a387db4
Parents: 3044b78
Author: Kitti Nanasi <kn...@cloudera.com>
Authored: Mon Dec 3 09:59:56 2018 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Dec 3 10:01:09 2018 -0800
----------------------------------------------------------------------
.../federation/metrics/NamenodeBeanMetrics.java | 5 +
.../server/blockmanagement/DatanodeManager.java | 7 +
.../hdfs/server/common/ECTopologyVerifier.java | 124 +++++++++++++++
.../namenode/ECTopologyVerifierResult.java | 45 ++++++
.../namenode/ErasureCodingPolicyManager.java | 9 ++
.../hdfs/server/namenode/FSNamesystem.java | 19 +++
.../hdfs/server/namenode/NameNodeMXBean.java | 7 +
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 51 +++++-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 35 +++++
.../hdfs/TestErasureCodingMultipleRacks.java | 18 +--
.../server/namenode/TestNameNodeMXBean.java | 19 +++
.../apache/hadoop/hdfs/tools/TestECAdmin.java | 157 +++++++++++++++++++
12 files changed, 479 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
index e8ebf0d..0ca5f73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -706,4 +706,9 @@ public class NamenodeBeanMetrics
public int getNumEncryptionZones() {
return 0;
}
+
+ @Override
+ public String getVerifyECWithTopologyResult() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 430c0d4..8a3f57b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1253,6 +1253,13 @@ public class DatanodeManager {
return getDatanodeListForReport(DatanodeReportType.DEAD).size();
}
+ /** @return the number of datanodes. */
+ public int getNumOfDataNodes() {
+ synchronized (this) {
+ return datanodeMap.size();
+ }
+ }
+
/** @return list of datanodes where decommissioning is in progress. */
public List<DatanodeDescriptor> getDecommissioningNodes() {
// There is no need to take namesystem reader lock as
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java
new file mode 100644
index 0000000..3591b2d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/ECTopologyVerifier.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.server.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.server.namenode.ECTopologyVerifierResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Class for verifying whether the cluster setup can support
+ * all enabled EC policies.
+ *
+ * Scenarios when the verification fails:
+ * 1. not enough data nodes compared to EC policy's highest data+parity number
+ * 2. not enough racks to satisfy BlockPlacementPolicyRackFaultTolerant
+ */
+@InterfaceAudience.Private
+public final class ECTopologyVerifier {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECTopologyVerifier.class);
+
+ private ECTopologyVerifier() {}
+
+ /**
+ * Verifies whether the cluster setup can support all enabled EC policies.
+ *
+ * @param report list of data node descriptors for all data nodes
+ * @param policies all system and user defined erasure coding policies
+ * @return the status of the verification
+ */
+ public static ECTopologyVerifierResult getECTopologyVerifierResult(
+ final DatanodeInfo[] report, final ErasureCodingPolicyInfo[] policies) {
+ final int numOfRacks = getNumberOfRacks(report);
+ return getECTopologyVerifierResult(policies, numOfRacks, report.length);
+ }
+
+ /**
+ * Verifies whether the cluster setup can support all enabled EC policies.
+ *
+ * @param policies all system and user defined erasure coding policies
+ * @param numOfRacks number of racks
+ * @param numOfDataNodes number of data nodes
+ * @return the status of the verification
+ */
+ public static ECTopologyVerifierResult getECTopologyVerifierResult(
+ final ErasureCodingPolicyInfo[] policies, final int numOfRacks,
+ final int numOfDataNodes) {
+ int minDN = 0;
+ int minRack = 0;
+ for (ErasureCodingPolicyInfo policy: policies) {
+ if (policy.isEnabled()) {
+ final int policyDN =
+ policy.getPolicy().getNumDataUnits() + policy.getPolicy()
+ .getNumParityUnits();
+ minDN = Math.max(minDN, policyDN);
+ final int policyRack = (int) Math.ceil(
+ policyDN / (double) policy.getPolicy().getNumParityUnits());
+ minRack = Math.max(minRack, policyRack);
+ }
+ }
+ if (minDN == 0 || minRack == 0) {
+ String resultMessage = "No erasure coding policy is enabled.";
+ LOG.trace(resultMessage);
+ return new ECTopologyVerifierResult(true, resultMessage);
+ }
+ return verifyECWithTopology(minDN, minRack, numOfRacks, numOfDataNodes);
+ }
+
+ private static ECTopologyVerifierResult verifyECWithTopology(
+ final int minDN, final int minRack,
+ final int numOfRacks, final int numOfDataNodes) {
+ String resultMessage;
+ if (numOfDataNodes < minDN) {
+ resultMessage = "The number of DataNodes (" + numOfDataNodes
+ + ") is less than the minimum required number of DataNodes ("
+ + minDN + ") for enabled erasure coding policy.";
+ LOG.debug(resultMessage);
+ return new ECTopologyVerifierResult(false, resultMessage);
+ }
+
+ if (numOfRacks < minRack) {
+ resultMessage = "The number of racks (" + numOfRacks
+ + ") is less than the minimum required number of racks ("
+ + minRack + ") for enabled erasure coding policy.";
+ LOG.debug(resultMessage);
+ return new ECTopologyVerifierResult(false, resultMessage);
+ }
+ return new ECTopologyVerifierResult(true,
+ "The cluster setup can support all enabled EC policies");
+ }
+
+ private static int getNumberOfRacks(DatanodeInfo[] report) {
+ final Map<String, Integer> racks = new HashMap<>();
+ for (DatanodeInfo dni : report) {
+ Integer count = racks.get(dni.getNetworkLocation());
+ if (count == null) {
+ count = 0;
+ }
+ racks.put(dni.getNetworkLocation(), count + 1);
+ }
+ return racks.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java
new file mode 100644
index 0000000..4757eab
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ECTopologyVerifierResult.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Result of the verification whether the current cluster setup can
+ * support all enabled EC policies.
+ */
+@InterfaceAudience.Private
+public class ECTopologyVerifierResult {
+
+ private final String resultMessage;
+ private final boolean isSupported;
+
+ public ECTopologyVerifierResult(boolean isSupported,
+ String resultMessage) {
+ this.resultMessage = resultMessage;
+ this.isSupported = isSupported;
+ }
+
+ public String getResultMessage() {
+ return resultMessage;
+ }
+
+ public boolean isSupported() {
+ return isSupported;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 57fa958..fc673f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -204,6 +205,14 @@ public final class ErasureCodingPolicyManager {
.toArray(new ErasureCodingPolicyInfo[0]);
}
+ public ErasureCodingPolicyInfo[] getCopyOfPolicies() {
+ ErasureCodingPolicyInfo[] copy;
+ synchronized (this) {
+ copy = Arrays.copyOf(allPolicies, allPolicies.length);
+ }
+ return copy;
+ }
+
/**
* Get a {@link ErasureCodingPolicy} by policy ID, including system policy
* and user defined policy.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8b1cdf6..d85782f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -102,6 +102,7 @@ import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.server.common.ECTopologyVerifier;
import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean;
import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
import org.apache.hadoop.util.Time;
@@ -8167,6 +8168,24 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.size();
}
+ @Override // NameNodeMXBean
+ public String getVerifyECWithTopologyResult() {
+ int numOfDataNodes = getBlockManager().getDatanodeManager()
+ .getNumOfDataNodes();
+ int numOfRacks = getBlockManager().getDatanodeManager()
+ .getNetworkTopology().getNumOfRacks();
+ ErasureCodingPolicyInfo[] ecPolicies =
+ getErasureCodingPolicyManager().getCopyOfPolicies();
+ ECTopologyVerifierResult result =
+ ECTopologyVerifier.getECTopologyVerifierResult(ecPolicies,
+ numOfRacks, numOfDataNodes);
+
+ Map<String, String> resultMap = new HashMap<String, String>();
+ resultMap.put("isSupported", Boolean.toString(result.isSupported()));
+ resultMap.put("resultMessage", result.getResultMessage());
+ return JSON.toString(resultMap);
+ }
+
// This method logs operatoinName without super user privilege.
// It should be called without holding FSN lock.
void checkSuperuserPrivilege(String operationName)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
index 5c7bbbb..8bf0bcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
@@ -303,4 +303,11 @@ public interface NameNodeMXBean {
*/
String getNameDirSize();
+ /**
+ * Verifies whether the cluster setup can support all enabled EC policies.
+ *
+ * @return the result of the verification
+ */
+ String getVerifyECWithTopologyResult();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 5f8626e..d63d9dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -24,9 +24,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NoECPolicySetException;
+import org.apache.hadoop.hdfs.server.common.ECTopologyVerifier;
+import org.apache.hadoop.hdfs.server.namenode.ECTopologyVerifierResult;
import org.apache.hadoop.hdfs.util.ECPolicyLoader;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.tools.TableListing;
@@ -588,6 +592,50 @@ public class ECAdmin extends Configured implements Tool {
}
}
+ /**
+ * Command to verify the cluster setup can support all enabled EC policies.
+ */
+ private static class VerifyClusterSetupCommand
+ implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-verifyClusterSetup";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + "]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ return getShortUsage() + "\n"
+ + "Verify the cluster setup can support all enabled erasure coding"
+ + " policies.\n";
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ final ErasureCodingPolicyInfo[] policies =
+ dfs.getClient().getNamenode().getErasureCodingPolicies();
+ final DatanodeInfo[] report = dfs.getClient().getNamenode()
+ .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL);
+
+ ECTopologyVerifierResult result = ECTopologyVerifier
+ .getECTopologyVerifierResult(report, policies);
+ System.out.println(result.getResultMessage());
+ if (result.isSupported()) {
+ return 0;
+ }
+ return 2;
+ }
+ }
+
private static final AdminHelper.Command[] COMMANDS = {
new ListECPoliciesCommand(),
new AddECPoliciesCommand(),
@@ -597,6 +645,7 @@ public class ECAdmin extends Configured implements Tool {
new UnsetECPolicyCommand(),
new ListECCodecsCommand(),
new EnableECPolicyCommand(),
- new DisableECPolicyCommand()
+ new DisableECPolicyCommand(),
+ new VerifyClusterSetupCommand()
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 9465b31..97609fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -2399,6 +2399,41 @@ public class DFSTestUtil {
}
/**
+ * Setup cluster with desired number of DN, racks, and specified number of
+ * rack that only has 1 DN. Other racks will be evenly setup with the number
+ * of DNs.
+ *
+ * @param conf the conf object to start the cluster.
+ * @param numDatanodes number of total Datanodes.
+ * @param numRacks number of total racks
+ * @param numSingleDnRacks number of racks that only has 1 DN
+ * @throws Exception
+ */
+ public static MiniDFSCluster setupCluster(final Configuration conf,
+ final int numDatanodes,
+ final int numRacks,
+ final int numSingleDnRacks)
+ throws Exception {
+ assert numDatanodes > numRacks;
+ assert numRacks > numSingleDnRacks;
+ assert numSingleDnRacks >= 0;
+ final String[] racks = new String[numDatanodes];
+ for (int i = 0; i < numSingleDnRacks; i++) {
+ racks[i] = "/rack" + i;
+ }
+ for (int i = numSingleDnRacks; i < numDatanodes; i++) {
+ racks[i] =
+ "/rack" + (numSingleDnRacks + (i % (numRacks - numSingleDnRacks)));
+ }
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numDatanodes)
+ .racks(racks)
+ .build();
+ cluster.waitActive();
+ return cluster;
+ }
+
+ /**
* Check the correctness of the snapshotDiff report.
* Make sure all items in the passed entries are in the snapshotDiff
* report.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java
index 3e87253..e47cbf0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java
@@ -92,23 +92,9 @@ public class TestErasureCodingMultipleRacks {
*/
public void setupCluster(final int numDatanodes, final int numRacks,
final int numSingleDnRacks) throws Exception {
- assert numDatanodes > numRacks;
- assert numRacks > numSingleDnRacks;
- assert numSingleDnRacks >= 0;
- final String[] racks = new String[numDatanodes];
- for (int i = 0; i < numSingleDnRacks; i++) {
- racks[i] = "/rack" + i;
- }
- for (int i = numSingleDnRacks; i < numDatanodes; i++) {
- racks[i] =
- "/rack" + (numSingleDnRacks + (i % (numRacks - numSingleDnRacks)));
- }
- cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(numDatanodes)
- .racks(racks)
- .build();
+ cluster = DFSTestUtil
+ .setupCluster(conf, numDatanodes, numRacks, numSingleDnRacks);
dfs = cluster.getFileSystem();
- cluster.waitActive();
dfs.setErasureCodingPolicy(new Path("/"), ecPolicy.getName());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 07a4945..99b1d2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -1008,6 +1008,7 @@ public class TestNameNodeMXBean {
expectedTotalReplicatedBlocks, totalReplicaBlocks.longValue());
assertEquals("Unexpected total ec block groups!",
expectedTotalECBlockGroups, totalECBlockGroups.longValue());
+ verifyEcClusterSetupVerifyResult(mbs);
}
private String getEnabledEcPoliciesMetric() throws Exception {
@@ -1017,4 +1018,22 @@ public class TestNameNodeMXBean {
return (String) (mbs.getAttribute(mxbeanName,
"EnabledEcPolicies"));
}
+
+ private void verifyEcClusterSetupVerifyResult(MBeanServer mbs)
+ throws Exception{
+ ObjectName namenodeMXBeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=NameNodeInfo");
+ String result = (String) mbs.getAttribute(namenodeMXBeanName,
+ "VerifyECWithTopologyResult");
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String, String> resultMap = mapper.readValue(result, Map.class);
+ Boolean isSupported = Boolean.parseBoolean(resultMap.get("isSupported"));
+ String resultMessage = resultMap.get("resultMessage");
+
+ assertFalse("Test cluster does not support all enabled " +
+ "erasure coding policies.", isSupported);
+ assertTrue(resultMessage.contains("The number of racks"));
+ assertTrue(resultMessage.contains("is less than the minimum required " +
+ "number of racks"));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd5e7c6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java
new file mode 100644
index 0000000..f9d0d40
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests some ECAdmin scenarios that are hard to test from
+ * {@link org.apache.hadoop.cli.TestErasureCodingCLI}.
+ */
+public class TestECAdmin {
+ public static final Logger LOG = LoggerFactory.getLogger(TestECAdmin.class);
+ private Configuration conf = new Configuration();
+ private MiniDFSCluster cluster;
+ private ECAdmin admin = new ECAdmin(conf);
+
+ private final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ private static final PrintStream OLD_OUT = System.out;
+
+ @Rule
+ public Timeout globalTimeout =
+ new Timeout(300000, TimeUnit.MILLISECONDS);
+
+ @Before
+ public void setup() throws Exception {
+ System.setOut(new PrintStream(out));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ System.out.flush();
+ System.err.flush();
+ out.reset();
+ } finally {
+ System.setOut(OLD_OUT);
+ }
+
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testRS63MinDN() throws Exception {
+ cluster = DFSTestUtil.setupCluster(conf, 6, 3, 0);
+ String[] args = {"-verifyClusterSetup"};
+ final int ret = admin.run(args);
+ LOG.info("Commend stdout: {}", out.toString());
+ assertEquals(2, ret);
+ assertTrue(out.toString()
+ .contains("less than the minimum required number of DataNodes"));
+ }
+
+ @Test
+ public void testRS104MinRacks() throws Exception {
+ cluster = DFSTestUtil.setupCluster(conf, 15, 3, 0);
+ cluster.getFileSystem().enableErasureCodingPolicy(
+ SystemErasureCodingPolicies
+ .getByID(SystemErasureCodingPolicies.RS_10_4_POLICY_ID).getName());
+ String[] args = {"-verifyClusterSetup"};
+ final int ret = admin.run(args);
+ LOG.info("Commend stdout: {}", out.toString());
+ assertEquals(2, ret);
+ assertTrue(out.toString()
+ .contains("less than the minimum required number of racks"));
+ }
+
+ @Test
+ public void testXOR21MinRacks() throws Exception {
+ cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0);
+ cluster.getFileSystem().disableErasureCodingPolicy(
+ SystemErasureCodingPolicies
+ .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName());
+ cluster.getFileSystem().enableErasureCodingPolicy(
+ SystemErasureCodingPolicies
+ .getByID(SystemErasureCodingPolicies.XOR_2_1_POLICY_ID).getName());
+ String[] args = {"-verifyClusterSetup"};
+ final int ret = admin.run(args);
+ LOG.info("Commend stdout: {}", out.toString());
+ assertEquals(2, ret);
+ assertTrue(out.toString()
+ .contains("less than the minimum required number of racks"));
+ }
+
+ @Test
+ public void testRS32MinRacks() throws Exception {
+ cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0);
+ cluster.getFileSystem().disableErasureCodingPolicy(
+ SystemErasureCodingPolicies
+ .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName());
+ cluster.getFileSystem().enableErasureCodingPolicy(
+ SystemErasureCodingPolicies
+ .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID).getName());
+ String[] args = {"-verifyClusterSetup"};
+ final int ret = admin.run(args);
+ LOG.info("Commend stdout: {}", out.toString());
+ assertEquals(2, ret);
+ assertTrue(out.toString()
+ .contains("less than the minimum required number of racks"));
+ }
+
+ @Test
+ public void testRS63Good() throws Exception {
+ cluster = DFSTestUtil.setupCluster(conf, 9, 3, 0);
+ String[] args = {"-verifyClusterSetup"};
+ final int ret = admin.run(args);
+ LOG.info("Commend stdout: {}", out.toString());
+ assertEquals(0, ret);
+ }
+
+ @Test
+ public void testNoECEnabled() throws Exception {
+ cluster = DFSTestUtil.setupCluster(conf, 9, 3, 0);
+ cluster.getFileSystem().disableErasureCodingPolicy(
+ SystemErasureCodingPolicies
+ .getByID(SystemErasureCodingPolicies.RS_6_3_POLICY_ID).getName());
+ String[] args = {"-verifyClusterSetup"};
+ final int ret = admin.run(args);
+ LOG.info("Commend stdout: {}", out.toString());
+ assertEquals(0, ret);
+ assertTrue(out.toString().contains("No erasure coding policy is enabled"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-14075. Terminate the namenode
when failed to start log segment. Contributed by Ayush Saxena.
Posted by sh...@apache.org.
HDFS-14075. Terminate the namenode when failed to start log segment. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042c8ef5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042c8ef5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042c8ef5
Branch: refs/heads/HDFS-12943
Commit: 042c8ef593ced1915a688e99aa9a6a52fdf66734
Parents: 8f3e12f
Author: Surendra Singh Lilhore <su...@apache.org>
Authored: Sun Dec 2 12:31:08 2018 +0530
Committer: Surendra Singh Lilhore <su...@apache.org>
Committed: Sun Dec 2 12:31:08 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSEditLog.java | 11 +++--
.../hadoop/hdfs/server/namenode/JournalSet.java | 10 ++++-
.../hadoop/hdfs/qjournal/TestNNWithQJM.java | 15 +++----
.../hdfs/server/namenode/TestEditLog.java | 8 +++-
.../namenode/TestEditLogJournalFailures.java | 44 ++++++++++++++++++++
5 files changed, 74 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 547ad57..56aa927 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -1382,10 +1382,15 @@ public class FSEditLog implements LogsPurgeable {
try {
editLogStream = journalSet.startLogSegment(segmentTxId, layoutVersion);
} catch (IOException ex) {
- throw new IOException("Unable to start log segment " +
- segmentTxId + ": too few journals successfully started.", ex);
+ final String msg = "Unable to start log segment " + segmentTxId
+ + ": too few journals successfully started.";
+ LOG.error(msg, ex);
+ synchronized (journalSetLock) {
+ IOUtils.cleanupWithLogger(LOG, journalSet);
+ }
+ terminate(1, msg);
}
-
+
curSegmentTxId = segmentTxId;
state = State.IN_SEGMENT;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
index 7be7073..4ab0828 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
@@ -76,7 +77,7 @@ public class JournalSet implements JournalManager {
* stream, then the stream will be aborted and set to null.
*/
static class JournalAndStream implements CheckableNameNodeResource {
- private final JournalManager journal;
+ private JournalManager journal;
private boolean disabled = false;
private EditLogOutputStream stream;
private final boolean required;
@@ -146,7 +147,12 @@ public class JournalSet implements JournalManager {
void setCurrentStreamForTests(EditLogOutputStream stream) {
this.stream = stream;
}
-
+
+ @VisibleForTesting
+ void setJournalForTests(JournalManager jm) {
+ this.journal = jm;
+ }
+
JournalManager getManager() {
return journal;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
index d713bc7..4483667 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ExitUtil.ExitException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -191,15 +192,15 @@ public class TestNNWithQJM {
// Start the NN - should fail because the JNs are still formatted
// with the old namespace ID.
try {
- cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(0)
- .manageNameDfsDirs(false)
- .format(false)
- .build();
+ ExitUtil.disableSystemExit();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
+ .manageNameDfsDirs(false).format(false).checkExitOnShutdown(false)
+ .build();
fail("New NN with different namespace should have been rejected");
- } catch (IOException ioe) {
+ } catch (ExitException ee) {
GenericTestUtils.assertExceptionContains(
- "Unable to start log segment 1: too few journals", ioe);
+ "Unable to start log segment 1: too few journals", ee);
+ assertTrue("Didn't terminate properly ", ExitUtil.terminateCalled());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index d24bffa..8eac143 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -81,6 +81,8 @@ import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ExitUtil.ExitException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
@@ -974,17 +976,19 @@ public class TestEditLog {
public void testFailedOpen() throws Exception {
File logDir = new File(TEST_DIR, "testFailedOpen");
logDir.mkdirs();
+ ExitUtil.disableSystemExit();
FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
try {
FileUtil.setWritable(logDir, false);
log.openForWrite(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
fail("Did no throw exception on only having a bad dir");
- } catch (IOException ioe) {
+ } catch (ExitException ee) {
GenericTestUtils.assertExceptionContains(
- "too few journals successfully started", ioe);
+ "too few journals successfully started", ee);
} finally {
FileUtil.setWritable(logDir, true);
log.close();
+ ExitUtil.resetFirstExitException();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042c8ef5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
index 1e8ee9c..07791da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogJournalFailures.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
@@ -34,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
@@ -262,6 +266,46 @@ public class TestEditLogJournalFailures {
}
}
+ @Test
+ public void testMultipleRedundantFailedEditsDirOnStartLogSegment()
+ throws Exception {
+ // Set up 4 name/edits dirs.
+ shutDownMiniCluster();
+ Configuration conf = getConf();
+ String[] nameDirs = new String[4];
+ for (int i = 0; i < nameDirs.length; i++) {
+ File nameDir = new File(PathUtils.getTestDir(getClass()), "name-dir" + i);
+ nameDir.mkdirs();
+ nameDirs[i] = nameDir.getAbsolutePath();
+ }
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+ StringUtils.join(nameDirs, ","));
+ conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY,
+ StringUtils.join(nameDirs, ",", 0, 3));
+
+ setUpMiniCluster(conf, false);
+
+ // All journals active.
+ assertTrue(doAnEdit());
+ // The NN has not terminated (no ExitException thrown)
+ spyOnJASjournal(3);
+ RemoteException re = intercept(RemoteException.class,
+ "too few journals successfully started.",
+ () -> ((DistributedFileSystem) fs).rollEdits());
+ GenericTestUtils.assertExceptionContains("ExitException", re);
+ }
+
+ private JournalManager spyOnJASjournal(int index) throws Exception {
+ JournalAndStream jas = getJournalAndStream(index);
+ JournalManager manager = jas.getManager();
+ JournalManager spyManager = spy(manager);
+ jas.setJournalForTests(spyManager);
+ doThrow(new IOException("Unable to start log segment ")).when(spyManager)
+ .startLogSegment(anyLong(), anyInt());
+
+ return spyManager;
+ }
+
/**
* Replace the journal at index <code>index</code> with one that throws an
* exception on flush.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDFS-13870. WebHDFS: Document
ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc. Contributed by Siyao Meng.
Posted by sh...@apache.org.
HDFS-13870. WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc. Contributed by Siyao Meng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e36e935
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e36e935
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e36e935
Branch: refs/heads/HDFS-12943
Commit: 0e36e935d909862401890d0a5410204504f48b31
Parents: bad1203
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Nov 30 11:31:34 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Nov 30 11:31:34 2018 +0800
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 24 ++++++++++++++++++++
1 file changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e36e935/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 383eda0..8661659 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -64,6 +64,8 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`SETTIMES`](#Set_Access_or_Modification_Time) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setTimes)
* [`RENEWDELEGATIONTOKEN`](#Renew_Delegation_Token) (see [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).renewDelegationToken)
* [`CANCELDELEGATIONTOKEN`](#Cancel_Delegation_Token) (see [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).cancelDelegationToken)
+ * [`ALLOWSNAPSHOT`](#Allow_Snapshot)
+ * [`DISALLOWSNAPSHOT`](#Disallow_Snapshot)
* [`CREATESNAPSHOT`](#Create_Snapshot) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).createSnapshot)
* [`RENAMESNAPSHOT`](#Rename_Snapshot) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).renameSnapshot)
* [`SETXATTR`](#Set_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setXAttr)
@@ -1302,6 +1304,28 @@ See also: [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).
Snapshot Operations
-------------------
+### Allow Snapshot
+
+* Submit a HTTP PUT request.
+
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=ALLOWSNAPSHOT"
+
+ The client receives a response with zero content length on success:
+
+ HTTP/1.1 200 OK
+ Content-Length: 0
+
+### Disallow Snapshot
+
+* Submit a HTTP PUT request.
+
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=DISALLOWSNAPSHOT"
+
+ The client receives a response with zero content length on success:
+
+ HTTP/1.1 200 OK
+ Content-Length: 0
+
### Create Snapshot
* Submit a HTTP PUT request.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-8975. [Submarine] Use
predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8.
(Zhankun Tang via wangda)
Posted by sh...@apache.org.
YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8. (Zhankun Tang via wangda)
Change-Id: If6c7904aa17895e543cfca245264249eb7328bdc
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89764392
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89764392
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89764392
Branch: refs/heads/HDFS-12943
Commit: 897643928c534062d45d00a36b95fd222299b4f6
Parents: 8ebeda9
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:39:06 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:39:06 2018 -0800
----------------------------------------------------------------------
.../submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89764392/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index b58ad77..2e84c96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -49,6 +49,7 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -218,7 +219,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
private String generateCommandLaunchScript(RunJobParameters parameters,
TaskType taskType, Component comp) throws IOException {
File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
- Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+ Writer w = new OutputStreamWriter(new FileOutputStream(file),
+ StandardCharsets.UTF_8);
PrintWriter pw = new PrintWriter(w);
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8974. Improve the assertion
message in TestGPUResourceHandler. (Zhankun Tang via wangda)
Posted by sh...@apache.org.
YARN-8974. Improve the assertion message in TestGPUResourceHandler. (Zhankun Tang via wangda)
Change-Id: I4eb58e9d251d5f54e7feffc4fbb813b4f5ae4b1b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ebeda98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ebeda98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ebeda98
Branch: refs/heads/HDFS-12943
Commit: 8ebeda98a9d3eac45598a33bae3d62e3ebb92cad
Parents: 9ed8756
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:36:30 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:36:30 2018 -0800
----------------------------------------------------------------------
.../linux/resources/gpu/TestGpuResourceHandler.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ebeda98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 10e5cd1..18785e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -465,7 +465,7 @@ public class TestGpuResourceHandler {
caughtException = true;
}
Assert.assertTrue(
- "Should fail since requested device Id is not in allowed list",
+ "Should fail since requested device Id is already assigned",
caughtException);
// Make sure internal state not changed.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HDFS-12943
Posted by sh...@apache.org.
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47d72601
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47d72601
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47d72601
Branch: refs/heads/HDFS-12943
Commit: 47d726015810d091f58387a0fc6bcc0ad30f4b02
Parents: b549707 1dabb31
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Wed Dec 5 13:38:45 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Wed Dec 5 13:38:45 2018 -0800
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 23 +-
.../authentication/util/KerberosName.java | 9 +-
.../TestKerberosAuthenticationHandler.java | 7 +-
.../authentication/util/TestKerberosName.java | 17 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 13 -
.../hadoop/fs/FileSystemMultipartUploader.java | 36 +-
.../org/apache/hadoop/fs/MultipartUploader.java | 88 ++-
.../hadoop/fs/MultipartUploaderFactory.java | 7 +
.../apache/hadoop/fs/shell/CopyCommands.java | 7 +-
.../java/org/apache/hadoop/security/KDiag.java | 46 +-
.../hadoop/security/LdapGroupsMapping.java | 163 ++--
.../src/main/resources/core-default.xml | 28 +-
.../src/site/markdown/GroupsMapping.md | 54 +-
.../src/site/markdown/SecureMode.md | 6 -
.../src/site/markdown/SingleCluster.md.vm | 3 -
.../src/site/markdown/filesystem/index.md | 1 +
.../markdown/filesystem/multipartuploader.md | 235 ++++++
.../AbstractContractMultipartUploaderTest.java | 565 +++++++++++---
.../TestLocalFSContractMultipartUploader.java | 10 +
.../hadoop/fs/shell/TestCopyPreserveFlag.java | 10 +
.../org/apache/hadoop/security/TestKDiag.java | 16 -
.../hadoop/security/TestLdapGroupsMapping.java | 80 +-
.../security/TestLdapGroupsMappingBase.java | 76 +-
.../TestLdapGroupsMappingWithFailover.java | 142 ++++
.../TestLdapGroupsMappingWithOneQuery.java | 16 +-
.../TestLdapGroupsMappingWithPosixGroup.java | 10 +-
.../security/TestUserGroupInformation.java | 27 +-
.../hdds/scm/storage/ChunkInputStream.java | 11 +
.../hdds/scm/storage/ChunkOutputStream.java | 21 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 8 +
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +
.../hdds/scm/container/ContainerInfo.java | 29 +-
.../scm/storage/ContainerProtocolCalls.java | 14 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 27 +-
.../org/apache/hadoop/ozone/audit/DNAction.java | 44 +-
.../apache/hadoop/ozone/common/Checksum.java | 249 ++++++
.../hadoop/ozone/common/ChecksumData.java | 190 +++++
.../ozone/common/OzoneChecksumException.java | 66 ++
.../container/common/helpers/ChunkInfo.java | 33 +-
.../hadoop/utils/db/ByteArrayKeyValue.java | 67 ++
.../java/org/apache/hadoop/utils/db/Codec.java | 38 +
.../apache/hadoop/utils/db/CodecRegistry.java | 70 ++
.../org/apache/hadoop/utils/db/DBStore.java | 12 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 12 +-
.../hadoop/utils/db/RDBStoreIterator.java | 20 +-
.../org/apache/hadoop/utils/db/RDBTable.java | 7 +-
.../org/apache/hadoop/utils/db/StringCodec.java | 45 ++
.../java/org/apache/hadoop/utils/db/Table.java | 63 +-
.../apache/hadoop/utils/db/TableIterator.java | 6 +-
.../org/apache/hadoop/utils/db/TypedTable.java | 184 +++++
.../main/proto/DatanodeContainerProtocol.proto | 21 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 20 +-
.../common/src/main/resources/ozone-default.xml | 54 ++
.../apache/hadoop/ozone/audit/DummyAction.java | 36 +-
.../hadoop/ozone/common/TestChecksum.java | 101 +++
.../hadoop/utils/db/TestDBStoreBuilder.java | 4 +-
.../apache/hadoop/utils/db/TestRDBStore.java | 31 +-
.../hadoop/utils/db/TestRDBTableStore.java | 4 +-
.../hadoop/utils/db/TestTypedRDBTableStore.java | 235 ++++++
.../container/common/impl/HddsDispatcher.java | 10 +-
.../common/interfaces/ContainerDispatcher.java | 5 +-
.../container/common/interfaces/Handler.java | 4 +-
.../transport/server/GrpcXceiverService.java | 3 +-
.../transport/server/XceiverServerGrpc.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 234 +++---
.../server/ratis/DispatcherContext.java | 133 ++++
.../server/ratis/XceiverServerRatis.java | 14 +-
.../container/keyvalue/KeyValueHandler.java | 91 ++-
.../container/keyvalue/helpers/BlockUtils.java | 8 +-
.../container/keyvalue/helpers/ChunkUtils.java | 54 +-
.../keyvalue/helpers/SmallFileUtils.java | 10 +-
.../keyvalue/impl/ChunkManagerImpl.java | 52 +-
.../keyvalue/interfaces/ChunkManager.java | 12 +-
.../common/impl/TestHddsDispatcher.java | 16 +-
.../keyvalue/TestChunkManagerImpl.java | 57 +-
.../container/keyvalue/TestKeyValueHandler.java | 48 +-
hadoop-hdds/docs/config.yaml | 5 +
hadoop-hdds/docs/content/Prometheus.md | 94 +++
.../hadoop/hdds/server/BaseHttpServer.java | 20 +
.../hdds/server/PrometheusMetricsSink.java | 133 ++++
.../hadoop/hdds/server/PrometheusServlet.java | 46 ++
.../hdds/server/TestPrometheusMetricsSink.java | 96 +++
hadoop-hdds/pom.xml | 1 +
.../scm/chillmode/PipelineChillModeRule.java | 108 +++
.../hdds/scm/chillmode/SCMChillModeManager.java | 19 +-
.../scm/container/ContainerReportHandler.java | 50 +-
.../scm/container/ContainerStateManager.java | 58 +-
.../IncrementalContainerReportHandler.java | 24 +-
.../hdds/scm/container/ReportHandlerHelper.java | 368 +++++++++
.../hadoop/hdds/scm/server/SCMMXBean.java | 5 +
.../scm/server/StorageContainerManager.java | 16 +-
.../scm/chillmode/TestSCMChillModeManager.java | 81 +-
.../hdds/scm/container/MockNodeManager.java | 1 +
.../container/TestContainerReportHandler.java | 772 +++++++++++++++----
.../container/TestContainerReportHelper.java | 113 +++
.../TestIncrementalContainerReportHandler.java | 218 ++++++
.../hadoop/hdfs/DFSOpsCountStatistics.java | 9 +
.../hadoop/hdfs/DistributedFileSystem.java | 18 +
.../federation/metrics/NamenodeBeanMetrics.java | 5 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../server/blockmanagement/BlockManager.java | 30 +-
.../BlockPlacementPolicyDefault.java | 22 +-
.../server/blockmanagement/DatanodeManager.java | 7 +
.../hdfs/server/common/ECTopologyVerifier.java | 124 +++
.../diskbalancer/planner/GreedyPlanner.java | 24 +-
.../namenode/ECTopologyVerifierResult.java | 45 ++
.../namenode/ErasureCodingPolicyManager.java | 9 +
.../server/namenode/FSDirStatAndListingOp.java | 9 +
.../hdfs/server/namenode/FSDirectory.java | 7 +
.../hadoop/hdfs/server/namenode/FSEditLog.java | 11 +-
.../hdfs/server/namenode/FSNamesystem.java | 19 +
.../hadoop/hdfs/server/namenode/JournalSet.java | 10 +-
.../hdfs/server/namenode/NameNodeMXBean.java | 7 +
.../hdfs/server/namenode/NamenodeFsck.java | 49 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 51 +-
.../OfflineImageViewerPB.java | 27 +-
.../offlineImageViewer/PBImageCorruption.java | 107 +++
.../PBImageCorruptionDetector.java | 344 +++++++++
.../PBImageDelimitedTextWriter.java | 38 +-
.../offlineImageViewer/PBImageTextWriter.java | 220 +++++-
.../src/main/resources/hdfs-default.xml | 10 +
.../src/site/markdown/HdfsImageViewer.md | 36 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 24 +
.../hdfs/TestHDFSContractMultipartUploader.java | 15 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 35 +
.../hadoop/hdfs/TestDistributedFileSystem.java | 63 +-
.../hdfs/TestErasureCodingMultipleRacks.java | 18 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 8 +
.../hadoop/hdfs/qjournal/TestNNWithQJM.java | 15 +-
.../hdfs/server/namenode/TestEditLog.java | 8 +-
.../namenode/TestEditLogJournalFailures.java | 44 ++
.../server/namenode/TestNameNodeMXBean.java | 19 +
.../apache/hadoop/hdfs/tools/TestECAdmin.java | 157 ++++
.../TestOfflineImageViewer.java | 219 +++++-
.../TestPBImageCorruption.java | 55 ++
.../test/resources/testMultipleCorruption.csv | 22 +
.../resources/testMultipleFileCorruption.csv | 21 +
.../test/resources/testSingleFileCorruption.csv | 18 +
.../resources/testSingleFolderCorruption.csv | 18 +
.../mapreduce/v2/app/TaskHeartbeatHandler.java | 34 +-
.../v2/app/TestTaskHeartbeatHandler.java | 53 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 8 +
.../mapreduce/jobhistory/EventWriter.java | 1 +
.../lib/output/FileOutputCommitter.java | 28 +-
.../src/main/resources/mapred-default.xml | 9 +
.../lib/output/TestFileOutputCommitter.java | 33 +
.../ozone/client/io/ChunkGroupOutputStream.java | 30 +-
.../hadoop/ozone/client/rpc/RpcClient.java | 26 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 54 +-
.../hadoop/ozone/om/OMMetadataManager.java | 17 +-
.../dist/src/main/compose/ozoneperf/README.md | 49 +-
.../src/main/compose/ozoneperf/compose-all.sh | 18 -
.../compose/ozoneperf/docker-compose-freon.yaml | 26 -
.../main/compose/ozoneperf/docker-compose.yaml | 23 +-
.../src/main/compose/ozoneperf/docker-config | 5 +-
.../dist/src/main/compose/ozoneperf/init.sh | 21 -
.../src/main/compose/ozoneperf/prometheus.yml | 11 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 98 ++-
.../ozone/container/ContainerTestHelper.java | 17 +-
.../common/TestBlockDeletingService.java | 3 +-
.../container/common/helpers/TestBlockData.java | 7 +-
.../common/impl/TestContainerPersistence.java | 73 +-
.../transport/server/ratis/TestCSMMetrics.java | 3 +-
.../container/server/TestContainerServer.java | 6 +-
.../hadoop/ozone/om/TestOzoneManager.java | 4 +-
.../apache/hadoop/ozone/scm/TestSCMMXBean.java | 81 +-
.../web/storage/DistributedStorageHandler.java | 26 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 82 +-
.../hadoop/ozone/web/ozShell/OzoneAddress.java | 6 +
.../ozone/web/ozShell/TestOzoneAddress.java | 3 +-
hadoop-ozone/pom.xml | 3 +-
.../ozone/s3/endpoint/ObjectEndpoint.java | 11 +-
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 +
.../genesis/BenchMarkDatanodeDispatcher.java | 16 +-
hadoop-project/pom.xml | 4 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 16 +-
.../hadoop/fs/s3a/S3AMultipartUploader.java | 31 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 2 +-
.../fs/s3a/s3guard/LocalMetadataEntry.java | 3 +
.../fs/s3a/s3guard/LocalMetadataStore.java | 29 +-
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 38 +-
.../s3a/ITestS3AContractMultipartUploader.java | 64 +-
.../fs/s3a/ITestS3ACopyFromLocalFile.java | 15 +-
.../fs/s3a/ITestS3GuardListConsistency.java | 18 +-
.../fs/s3a/TestS3AMultipartUploaderSupport.java | 2 +-
.../s3a/s3guard/ITestDynamoDBMetadataStore.java | 18 +-
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 15 +
.../fs/azure/AzureNativeFileSystemStore.java | 62 +-
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 39 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 13 +-
.../azure/ITestNativeAzureFileSystemLive.java | 4 +-
.../fs/azure/TestKeyPageBlobDirectories.java | 170 ++++
.../hadoop/yarn/service/webapp/ApiServer.java | 209 +++--
.../hadoop/yarn/service/ServiceClientTest.java | 2 +-
.../hadoop/yarn/service/ServiceScheduler.java | 6 +-
.../yarn/service/client/ServiceClient.java | 1 +
.../yarn/service/component/Component.java | 4 +-
.../component/instance/ComponentInstance.java | 57 +-
.../instance/ComponentInstanceState.java | 3 +-
.../yarn/service/monitor/ServiceMonitor.java | 5 +-
.../hadoop/yarn/service/TestServiceManager.java | 22 +-
.../yarn/service/TestYarnNativeServices.java | 3 +-
.../yarn/service/component/TestComponent.java | 8 +
.../instance/TestComponentInstance.java | 39 +-
.../installation/install.conf | 74 --
.../installation/install.sh | 116 ---
.../package/calico/calico-node.service | 50 --
.../installation/package/calico/calicoctl.cfg | 22 -
.../installation/package/docker/daemon.json | 23 -
.../installation/package/docker/docker.service | 35 -
.../installation/package/etcd/etcd.service | 40 -
.../package/hadoop/container-executor.cfg | 41 -
.../installation/package/submarine/submarine.sh | 25 -
.../installation/scripts/calico.sh | 224 ------
.../installation/scripts/docker.sh | 166 ----
.../installation/scripts/download-server.sh | 42 -
.../installation/scripts/environment.sh | 213 -----
.../installation/scripts/etcd.sh | 152 ----
.../installation/scripts/hadoop.sh | 117 ---
.../installation/scripts/menu.sh | 444 -----------
.../installation/scripts/nvidia-docker.sh | 99 ---
.../installation/scripts/nvidia.sh | 120 ---
.../installation/scripts/submarine.sh | 38 -
.../installation/scripts/utils.sh | 123 ---
.../yarnservice/YarnServiceJobSubmitter.java | 4 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 17 +
.../hadoop/yarn/client/cli/ClusterCLI.java | 2 +
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 105 ++-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 2 +
.../apache/hadoop/yarn/client/cli/QueueCLI.java | 2 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 1 +
.../apache/hadoop/yarn/client/cli/YarnCLI.java | 9 +-
.../hadoop/yarn/client/cli/TestClusterCLI.java | 41 +-
.../hadoop/yarn/client/cli/TestLogsCLI.java | 37 +
.../hadoop/yarn/client/cli/TestYarnCLI.java | 89 ++-
.../api/impl/TimelineReaderClientImpl.java | 2 +
.../LogAggregationFileController.java | 74 +-
.../hadoop-yarn/hadoop-yarn-csi/pom.xml | 78 +-
.../utils/FederationStateStoreFacade.java | 5 +-
.../utils/FederationStateStoreTestUtil.java | 2 +-
.../utils/TestFederationStateStoreFacade.java | 17 +
.../launcher/ContainerLaunch.java | 2 +-
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../runtime/DockerLinuxContainerRuntime.java | 70 +-
.../linux/runtime/docker/DockerRunCommand.java | 6 +
.../resourceplugin/ResourcePluginManager.java | 14 +-
.../deviceframework/DeviceMappingManager.java | 324 ++++++++
.../deviceframework/DevicePluginAdapter.java | 20 +-
.../DeviceResourceHandlerImpl.java | 145 ++++
.../impl/modules/fpga/fpga-module.c | 5 +-
.../impl/modules/gpu/gpu-module.c | 5 +-
.../container-executor/impl/utils/docker-util.c | 72 ++
.../container-executor/impl/utils/docker-util.h | 1 +
.../test/utils/test_docker_util.cc | 94 ++-
.../launcher/TestContainerLaunch.java | 2 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 38 +
.../resources/gpu/TestGpuResourceHandler.java | 2 +-
.../runtime/TestDockerContainerRuntime.java | 15 +-
.../runtime/docker/TestDockerRunCommand.java | 9 +-
.../TestDeviceMappingManager.java | 366 +++++++++
.../TestDevicePluginAdapter.java | 388 +++++++++-
.../placement/AppNameMappingPlacementRule.java | 12 +-
.../placement/PlacementRule.java | 4 +-
.../UserGroupMappingPlacementRule.java | 11 +-
.../scheduler/fair/FSPreemptionThread.java | 55 +-
.../resourcemanager/webapp/NodesPage.java | 4 +-
.../webapp/dao/SchedulerInfo.java | 2 +
.../fair/TestFairSchedulerPreemption.java | 155 ++--
.../src/site/markdown/DockerContainers.md | 1 +
269 files changed, 10395 insertions(+), 4277 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47d72601/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47d72601/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47d72601/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-15974. Upgrade Curator
version to 2.13.0 to fix ZK tests. Contributed by Akira Ajisaka
Posted by sh...@apache.org.
HADOOP-15974. Upgrade Curator version to 2.13.0 to fix ZK tests. Contributed by Akira Ajisaka
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9287ab36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9287ab36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9287ab36
Branch: refs/heads/HDFS-12943
Commit: 9287ab364292ce917fc120532681131821ac53ef
Parents: aa89492
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Dec 4 15:44:03 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Dec 4 15:44:03 2018 -0600
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9287ab36/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 5fd3a56..2b1fc09 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -87,7 +87,7 @@
<protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>
<zookeeper.version>3.4.13</zookeeper.version>
- <curator.version>2.12.0</curator.version>
+ <curator.version>2.13.0</curator.version>
<findbugs.version>3.0.0</findbugs.version>
<spotbugs.version>3.1.0-RC1</spotbugs.version>
<dnsjava.version>2.1.7</dnsjava.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-14105. Replace TreeSet in
NamenodeFsck with HashSet. Contributed by Beluga Behr.
Posted by sh...@apache.org.
HDFS-14105. Replace TreeSet in NamenodeFsck with HashSet. Contributed by Beluga Behr.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99e201df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99e201df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99e201df
Branch: refs/heads/HDFS-12943
Commit: 99e201dfe2295be830efcc80be34706802da30be
Parents: b09cfad
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 30 11:07:11 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 30 11:07:11 2018 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/NamenodeFsck.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99e201df/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index b4c0a93..095a6ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -30,10 +30,11 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.TreeSet;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.io.IOUtils;
@@ -1021,10 +1022,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
* around.
*/
private void copyBlock(final DFSClient dfs, LocatedBlock lblock,
- OutputStream fos) throws Exception {
+ OutputStream fos) throws Exception {
int failures = 0;
InetSocketAddress targetAddr = null;
- TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
+ Set<DatanodeInfo> deadNodes = new HashSet<DatanodeInfo>();
BlockReader blockReader = null;
ExtendedBlock block = lblock.getBlock();
@@ -1132,9 +1133,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
* That's the local one, if available.
*/
private DatanodeInfo bestNode(DFSClient dfs, DatanodeInfo[] nodes,
- TreeSet<DatanodeInfo> deadNodes) throws IOException {
- if ((nodes == null) ||
- (nodes.length - deadNodes.size() < 1)) {
+ Set<DatanodeInfo> deadNodes) throws IOException {
+ if ((nodes == null) || (nodes.length - deadNodes.size() < 1)) {
throw new IOException("No live nodes contain current block");
}
DatanodeInfo chosenNode;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HADOOP-15968. ABFS: add try catch
for UGI failure when initializing ABFS.
Posted by sh...@apache.org.
HADOOP-15968. ABFS: add try catch for UGI failure when initializing ABFS.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8bbd818
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8bbd818
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8bbd818
Branch: refs/heads/HDFS-12943
Commit: a8bbd818d5bc4762324bcdb7cf1fdd5c2f93891b
Parents: de42555
Author: Da Zhou <da...@microsoft.com>
Authored: Tue Dec 4 13:39:10 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 4 13:39:10 2018 +0000
----------------------------------------------------------------------
.../apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 7 ++++++-
.../hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 13 +++++++++----
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8bbd818/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 38b0c77..b4277c2 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -121,7 +121,12 @@ public class AzureBlobFileSystem extends FileSystem {
}
if (!abfsConfiguration.getSkipUserGroupMetadataDuringInitialization()) {
- this.primaryUserGroup = userGroupInformation.getPrimaryGroupName();
+ try {
+ this.primaryUserGroup = userGroupInformation.getPrimaryGroupName();
+ } catch (IOException ex) {
+ LOG.error("Failed to get primary group for {}, using user name as primary group name", user);
+ this.primaryUserGroup = this.user;
+ }
} else {
//Provide a default group name
this.primaryUserGroup = this.user;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8bbd818/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index fc684ac..5f1692f 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -102,8 +102,8 @@ public class AzureBlobFileSystemStore {
private AbfsClient client;
private URI uri;
private final UserGroupInformation userGroupInformation;
- private final String userName;
- private final String primaryUserGroup;
+ private String userName;
+ private String primaryUserGroup;
private static final String DATE_TIME_PATTERN = "E, dd MMM yyyy HH:mm:ss 'GMT'";
private static final String XMS_PROPERTIES_ENCODING = "ISO-8859-1";
private static final int LIST_MAX_RESULTS = 5000;
@@ -134,10 +134,15 @@ public class AzureBlobFileSystemStore {
this.userName = userGroupInformation.getShortUserName();
if (!abfsConfiguration.getSkipUserGroupMetadataDuringInitialization()) {
- primaryUserGroup = userGroupInformation.getPrimaryGroupName();
+ try {
+ this.primaryUserGroup = userGroupInformation.getPrimaryGroupName();
+ } catch (IOException ex) {
+ LOG.error("Failed to get primary group for {}, using user name as primary group name", userName);
+ this.primaryUserGroup = userName;
+ }
} else {
//Provide a default group name
- primaryUserGroup = userName;
+ this.primaryUserGroup = userName;
}
this.azureAtomicRenameDirSet = new HashSet<>(Arrays.asList(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-9061. Improve the GPU/FPGA
module log message of container-executor. (Zhankun Tang via wangda)
Posted by sh...@apache.org.
YARN-9061. Improve the GPU/FPGA module log message of container-executor. (Zhankun Tang via wangda)
Change-Id: Iece9b47438357077a53984a820d4d6423f480518
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed87567
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed87567
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed87567
Branch: refs/heads/HDFS-12943
Commit: 9ed87567ad0f1c26a263ce6d8fba56d066260c5c
Parents: 579ef4b
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:31:31 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:31:31 2018 -0800
----------------------------------------------------------------------
.../native/container-executor/impl/modules/fpga/fpga-module.c | 5 +++--
.../native/container-executor/impl/modules/gpu/gpu-module.c | 5 +++--
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
index c1a2f83..e947d7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
@@ -141,7 +141,7 @@ void reload_fpga_configuration() {
/*
* Format of FPGA request commandline:
*
- * c-e fpga --excluded_fpgas 0,1,3 --container_id container_x_y
+ * c-e --module-fpga --excluded_fpgas 0,1,3 --container_id container_x_y
*/
int handle_fpga_request(update_cgroups_parameters_function func,
const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_fpga_request(update_cgroups_parameters_function func,
if (!minor_devices) {
// Minor devices is null, skip following call.
- fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+ fprintf(ERRORFILE,
+ "--excluded-fpgas is not specified, skip cgroups call.\n");
goto cleanup;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
index 1a1b164..7522338 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
@@ -141,7 +141,7 @@ void reload_gpu_configuration() {
/*
* Format of GPU request commandline:
*
- * c-e gpu --excluded_gpus 0,1,3 --container_id container_x_y
+ * c-e --module-gpu --excluded_gpus 0,1,3 --container_id container_x_y
*/
int handle_gpu_request(update_cgroups_parameters_func func,
const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_gpu_request(update_cgroups_parameters_func func,
if (!minor_devices) {
// Minor devices is null, skip following call.
- fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+ fprintf(ERRORFILE,
+ "--excluded_gpus is not specified, skip cgroups call.\n");
goto cleanup;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDDS-894. Content-length should be
set for ozone s3 ranged download. Contributed by Elek Marton.
Posted by sh...@apache.org.
HDDS-894. Content-length should be set for ozone s3 ranged download. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de425550
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de425550
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de425550
Branch: refs/heads/HDFS-12943
Commit: de4255509adbd15fbbf9ade245ae6bb6db8b36b7
Parents: fb9deed
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Dec 3 15:17:44 2018 -0800
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Dec 3 15:17:44 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 11 ++++++++---
.../apache/hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 ++
2 files changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de425550/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index c504387..fdcadfc 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
+import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
import static javax.ws.rs.core.HttpHeaders.LAST_MODIFIED;
import org.apache.commons.io.IOUtils;
import static org.apache.hadoop.ozone.s3.util.S3Consts.ACCEPT_RANGE_HEADER;
@@ -204,10 +205,12 @@ public class ObjectEndpoint extends EndpointBase {
IOUtils.copy(key, dest);
}
};
- responseBuilder = Response.ok(output);
+ responseBuilder = Response
+ .ok(output)
+ .header(CONTENT_LENGTH, keyDetails.getDataSize());
} else {
- LOG.info("range Header provided value is {}", rangeHeader);
+ LOG.debug("range Header provided value is {}", rangeHeader);
OzoneInputStream key = bucket.readKey(keyPath);
long startOffset = rangeHeader.getStartOffset();
@@ -229,7 +232,9 @@ public class ObjectEndpoint extends EndpointBase {
copyLength);
}
};
- responseBuilder = Response.ok(output);
+ responseBuilder = Response
+ .ok(output)
+ .header(CONTENT_LENGTH, copyLength);
String contentRangeVal = RANGE_HEADER_SUPPORTED_UNIT + " " +
rangeHeader.getStartOffset() + "-" + rangeHeader.getEndOffset() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de425550/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
index f9df9aa..2455322 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
@@ -80,6 +80,8 @@ public class TestObjectGet {
IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8"));
Assert.assertEquals(CONTENT, keyContent);
+ Assert.assertEquals("" + keyContent.length(),
+ response.getHeaderString("Content-Length"));
DateTimeFormatter.RFC_1123_DATE_TIME
.parse(response.getHeaderString("Last-Modified"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-9010. Fix the incorrect
trailing slash deletion in constructor method of CGroupsHandlerImpl. (Zhankun
Tang via wangda)
Posted by sh...@apache.org.
YARN-9010. Fix the incorrect trailing slash deletion in constructor method of CGroupsHandlerImpl. (Zhankun Tang via wangda)
Change-Id: Iaecc66d57781cc10f19ead4647e47fc9556676da
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bad12031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bad12031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bad12031
Branch: refs/heads/HDFS-12943
Commit: bad12031f603347a701249a1e3ef5d879a5f1c8f
Parents: 0081b02
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 29 14:56:07 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Nov 29 14:56:07 2018 -0800
----------------------------------------------------------------------
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 38 ++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad12031/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 050d0a8..1b2c780 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -87,9 +87,10 @@ class CGroupsHandlerImpl implements CGroupsHandler {
CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor
privilegedOperationExecutor, String mtab)
throws ResourceHandlerException {
+ // Remove leading and trialing slash(es)
this.cGroupPrefix = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn")
- .replaceAll("^/", "").replaceAll("$/", "");
+ .replaceAll("^/+", "").replaceAll("/+$", "");
this.enableCGroupMount = conf.getBoolean(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
this.cGroupMountPath = conf.get(YarnConfiguration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad12031/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index ea6fb52..70badaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -598,4 +598,42 @@ public class TestCGroupsHandlerImpl {
FileUtils.deleteQuietly(cpu);
}
}
+
+ // Remove leading and trailing slashes
+ @Test
+ public void testCgroupsHierarchySetting() throws ResourceHandlerException {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath);
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ "/hadoop-yarn");
+ CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+ String expectedRelativePath = "hadoop-yarn/c1";
+ Assert.assertEquals(expectedRelativePath,
+ cGroupsHandler.getRelativePathForCGroup("c1"));
+
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ "hadoop-yarn");
+ cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+ Assert.assertEquals(expectedRelativePath,
+ cGroupsHandler.getRelativePathForCGroup("c1"));
+
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ "hadoop-yarn/");
+ cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+ Assert.assertEquals(expectedRelativePath,
+ cGroupsHandler.getRelativePathForCGroup("c1"));
+
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ "//hadoop-yarn//");
+ cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+ Assert.assertEquals(expectedRelativePath,
+ cGroupsHandler.getRelativePathForCGroup("c1"));
+
+ expectedRelativePath = "hadoop-yarn/root/c1";
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+ "//hadoop-yarn/root//");
+ cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+ Assert.assertEquals(expectedRelativePath,
+ cGroupsHandler.getRelativePathForCGroup("c1"));
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-8882. [YARN-8851] Add a
shared device mapping manager (scheduler) for device plugins. (Zhankun Tang
via wangda)
Posted by sh...@apache.org.
YARN-8882. [YARN-8851] Add a shared device mapping manager (scheduler) for device plugins. (Zhankun Tang via wangda)
Change-Id: I9435136642c3d556971a357bf687f69df90bb45e
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/579ef4be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/579ef4be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/579ef4be
Branch: refs/heads/HDFS-12943
Commit: 579ef4be063745c5211127eca83a393ceddc8b79
Parents: 9de8e8d
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Nov 28 14:09:52 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Nov 28 14:09:52 2018 -0800
----------------------------------------------------------------------
.../resourceplugin/ResourcePluginManager.java | 14 +-
.../deviceframework/DeviceMappingManager.java | 324 ++++++++++++++++
.../deviceframework/DevicePluginAdapter.java | 20 +-
.../DeviceResourceHandlerImpl.java | 145 +++++++
.../TestDeviceMappingManager.java | 366 +++++++++++++++++
.../TestDevicePluginAdapter.java | 388 ++++++++++++++++++-
6 files changed, 1245 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index 9741b12..6dfe817 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DeviceMappingManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DevicePluginAdapter;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
@@ -52,12 +53,13 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
public class ResourcePluginManager {
private static final Logger LOG =
LoggerFactory.getLogger(ResourcePluginManager.class);
- private static final Set<String> SUPPORTED_RESOURCE_PLUGINS = ImmutableSet.of(
- GPU_URI, FPGA_URI);
+ private static final Set<String> SUPPORTED_RESOURCE_PLUGINS =
+ ImmutableSet.of(GPU_URI, FPGA_URI);
private Map<String, ResourcePlugin> configuredPlugins =
Collections.emptyMap();
+ private DeviceMappingManager deviceMappingManager = null;
public synchronized void initialize(Context context)
throws YarnException, ClassNotFoundException {
@@ -123,7 +125,7 @@ public class ResourcePluginManager {
throws YarnRuntimeException, ClassNotFoundException {
LOG.info("The pluggable device framework enabled," +
"trying to load the vendor plugins");
-
+ deviceMappingManager = new DeviceMappingManager(context);
String[] pluginClassNames = configuration.getStrings(
YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
if (null == pluginClassNames) {
@@ -174,7 +176,7 @@ public class ResourcePluginManager {
resourceName,
pluginClassName);
DevicePluginAdapter pluginAdapter = new DevicePluginAdapter(
- resourceName, dpInstance);
+ resourceName, dpInstance, deviceMappingManager);
LOG.info("Adapter of {} created. Initializing..", pluginClassName);
try {
pluginAdapter.initialize(context);
@@ -235,6 +237,10 @@ public class ResourcePluginManager {
return true;
}
+ public DeviceMappingManager getDeviceMappingManager() {
+ return deviceMappingManager;
+ }
+
public synchronized void cleanup() throws YarnException {
for (ResourcePlugin plugin : configuredPlugins.values()) {
plugin.cleanup();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java
new file mode 100644
index 0000000..b8b711b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Collections;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Schedule device resource based on requirements and do book keeping
+ * It holds all device type resource and can do scheduling as a default
+ * scheduler.
+ * */
+public class DeviceMappingManager {
+ static final Log LOG = LogFactory.getLog(DeviceMappingManager.class);
+
+ private Context nmContext;
+ private static final int WAIT_MS_PER_LOOP = 1000;
+
+ /**
+ * Hold all type of devices.
+ * key is the device resource name
+ * value is a sorted set of {@link Device}
+ * */
+ private Map<String, Set<Device>> allAllowedDevices =
+ new ConcurrentHashMap<>();
+
+ /**
+ * Hold used devices.
+ * key is the device resource name
+ * value is a sorted map of {@link Device} and {@link ContainerId} pairs
+ * */
+ private Map<String, Map<Device, ContainerId>> allUsedDevices =
+ new ConcurrentHashMap<>();
+
+ public DeviceMappingManager(Context context) {
+ nmContext = context;
+ }
+
+ @VisibleForTesting
+ public Map<String, Set<Device>> getAllAllowedDevices() {
+ return allAllowedDevices;
+ }
+
+ @VisibleForTesting
+ public Map<String, Map<Device, ContainerId>> getAllUsedDevices() {
+ return allUsedDevices;
+ }
+
+ public synchronized void addDeviceSet(String resourceName,
+ Set<Device> deviceSet) {
+ LOG.info("Adding new resource: " + "type:"
+ + resourceName + "," + deviceSet);
+ allAllowedDevices.put(resourceName, new TreeSet<>(deviceSet));
+ allUsedDevices.put(resourceName, new TreeMap<>());
+ }
+
+ public DeviceAllocation assignDevices(String resourceName,
+ Container container)
+ throws ResourceHandlerException {
+ DeviceAllocation allocation = internalAssignDevices(resourceName,
+ container);
+ // Wait for a maximum of 120 seconds if no available Devices are there
+ // which are yet to be released.
+ final int timeoutMsecs = 120 * WAIT_MS_PER_LOOP;
+ int timeWaiting = 0;
+ while (allocation == null) {
+ if (timeWaiting >= timeoutMsecs) {
+ break;
+ }
+
+ // Sleep for 1 sec to ensure there are some free devices which are
+ // getting released.
+ try {
+ LOG.info("Container : " + container.getContainerId()
+ + " is waiting for free " + resourceName + " devices.");
+ Thread.sleep(WAIT_MS_PER_LOOP);
+ timeWaiting += WAIT_MS_PER_LOOP;
+ allocation = internalAssignDevices(resourceName, container);
+ } catch (InterruptedException e) {
+ // On any interrupt, break the loop and continue execution.
+ break;
+ }
+ }
+
+ if (allocation == null) {
+ String message = "Could not get valid " + resourceName
+ + " device for container '" + container.getContainerId()
+ + "' as some other containers might not releasing them.";
+ LOG.warn(message);
+ throw new ResourceHandlerException(message);
+ }
+ return allocation;
+ }
+
+ private synchronized DeviceAllocation internalAssignDevices(
+ String resourceName, Container container)
+ throws ResourceHandlerException {
+ Resource requestedResource = container.getResource();
+ ContainerId containerId = container.getContainerId();
+ int requestedDeviceCount = getRequestedDeviceCount(resourceName,
+ requestedResource);
+ LOG.debug("Try allocating " + requestedDeviceCount
+ + " " + resourceName);
+ // Assign devices to container if requested some.
+ if (requestedDeviceCount > 0) {
+ if (requestedDeviceCount > getAvailableDevices(resourceName)) {
+ // If there are some devices which are getting released, wait for few
+ // seconds to get it.
+ if (requestedDeviceCount <= getReleasingDevices(resourceName)
+ + getAvailableDevices(resourceName)) {
+ return null;
+ }
+ }
+
+ int availableDeviceCount = getAvailableDevices(resourceName);
+ if (requestedDeviceCount > availableDeviceCount) {
+ throw new ResourceHandlerException("Failed to find enough "
+ + resourceName
+ + ", requestor=" + containerId
+ + ", #Requested=" + requestedDeviceCount + ", #available="
+ + availableDeviceCount);
+ }
+
+ Set<Device> assignedDevices = new TreeSet<>();
+ Map<Device, ContainerId> usedDevices = allUsedDevices.get(resourceName);
+ Set<Device> allowedDevices = allAllowedDevices.get(resourceName);
+
+ defaultScheduleAction(allowedDevices, usedDevices,
+ assignedDevices, containerId, requestedDeviceCount);
+
+ // Record in state store if we allocated anything
+ if (!assignedDevices.isEmpty()) {
+ try {
+ // Update state store.
+ nmContext.getNMStateStore().storeAssignedResources(container,
+ resourceName,
+ new ArrayList<>(assignedDevices));
+ } catch (IOException e) {
+ cleanupAssignedDevices(resourceName, containerId);
+ throw new ResourceHandlerException(e);
+ }
+ }
+
+ return new DeviceAllocation(resourceName, assignedDevices,
+ Sets.difference(allowedDevices, assignedDevices));
+ }
+ return new DeviceAllocation(resourceName, null,
+ allAllowedDevices.get(resourceName));
+ }
+
+ public synchronized void recoverAssignedDevices(String resourceName,
+ ContainerId containerId)
+ throws ResourceHandlerException {
+ Container c = nmContext.getContainers().get(containerId);
+ Map<Device, ContainerId> usedDevices = allUsedDevices.get(resourceName);
+ Set<Device> allowedDevices = allAllowedDevices.get(resourceName);
+ if (null == c) {
+ throw new ResourceHandlerException(
+ "This shouldn't happen, cannot find container with id="
+ + containerId);
+ }
+
+ for (Serializable deviceSerializable : c.getResourceMappings()
+ .getAssignedResources(resourceName)) {
+ if (!(deviceSerializable instanceof Device)) {
+ throw new ResourceHandlerException(
+ "Trying to recover device id, however it"
+ + " is not Device instance, this shouldn't happen");
+ }
+
+ Device device = (Device) deviceSerializable;
+
+ // Make sure it is in allowed device.
+ if (!allowedDevices.contains(device)) {
+ throw new ResourceHandlerException(
+ "Try to recover device = " + device
+ + " however it is not in allowed device list:" + StringUtils
+ .join(",", allowedDevices));
+ }
+
+ // Make sure it is not occupied by anybody else
+ if (usedDevices.containsKey(device)) {
+ throw new ResourceHandlerException(
+ "Try to recover device id = " + device
+ + " however it is already assigned to container="
+ + usedDevices.get(device)
+ + ", please double check what happened.");
+ }
+
+ usedDevices.put(device, containerId);
+ }
+ }
+
+ public synchronized void cleanupAssignedDevices(String resourceName,
+ ContainerId containerId) {
+ Iterator<Map.Entry<Device, ContainerId>> iter =
+ allUsedDevices.get(resourceName).entrySet().iterator();
+ while (iter.hasNext()) {
+ if (iter.next().getValue().equals(containerId)) {
+ iter.remove();
+ }
+ }
+ }
+
+ public static int getRequestedDeviceCount(String resourceName,
+ Resource requestedResource) {
+ try {
+ return Long.valueOf(requestedResource.getResourceValue(
+ resourceName)).intValue();
+ } catch (ResourceNotFoundException e) {
+ return 0;
+ }
+ }
+
+ public int getAvailableDevices(String resourceName) {
+ return allAllowedDevices.get(resourceName).size()
+ - allUsedDevices.get(resourceName).size();
+ }
+
+ private long getReleasingDevices(String resourceName) {
+ long releasingDevices = 0;
+ Map<Device, ContainerId> used = allUsedDevices.get(resourceName);
+ Iterator<Map.Entry<Device, ContainerId>> iter = used.entrySet()
+ .iterator();
+ while (iter.hasNext()) {
+ ContainerId containerId = iter.next().getValue();
+ Container container = nmContext.getContainers().get(containerId);
+ if (container != null) {
+ if (container.isContainerInFinalStates()) {
+ releasingDevices = releasingDevices + container.getResource()
+ .getResourceInformation(resourceName).getValue();
+ }
+ }
+ }
+ return releasingDevices;
+ }
+
+ // default scheduling logic
+ private void defaultScheduleAction(Set<Device> allowed,
+ Map<Device, ContainerId> used, Set<Device> assigned,
+ ContainerId containerId, int count) {
+ LOG.debug("Using default scheduler. Allowed:" + allowed
+ + ",Used:" + used + ", containerId:" + containerId);
+ for (Device device : allowed) {
+ if (!used.containsKey(device)) {
+ used.put(device, containerId);
+ assigned.add(device);
+ if (assigned.size() == count) {
+ return;
+ }
+ }
+ } // end for
+ }
+
+ static class DeviceAllocation {
+ private String resourceName;
+
+ private Set<Device> allowed = Collections.emptySet();
+ private Set<Device> denied = Collections.emptySet();
+
+ DeviceAllocation(String resName, Set<Device> a,
+ Set<Device> d) {
+ this.resourceName = resName;
+ if (a != null) {
+ this.allowed = ImmutableSet.copyOf(a);
+ }
+ if (d != null) {
+ this.denied = ImmutableSet.copyOf(d);
+ }
+ }
+
+
+ public Set<Device> getAllowed() {
+ return allowed;
+ }
+
+ @Override
+ public String toString() {
+ return "ResourceType: " + resourceName
+ + ", Allowed Devices: " + allowed
+ + ", Denied Devices: " + denied;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
index 18a6992..1636cb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
/**
- * The {@link DevicePluginAdapter} will adapt existing hooks
+ * The {@link DevicePluginAdapter} will adapt existing hooks.
* into vendor plugin's logic.
* It decouples the vendor plugin from YARN's device framework
*
@@ -43,13 +43,21 @@ public class DevicePluginAdapter implements ResourcePlugin {
private final String resourceName;
private final DevicePlugin devicePlugin;
+ private DeviceMappingManager deviceMappingManager;
private DeviceResourceUpdaterImpl deviceResourceUpdater;
+ private DeviceResourceHandlerImpl deviceResourceHandler;
- public DevicePluginAdapter(String name, DevicePlugin dp) {
+ public DevicePluginAdapter(String name, DevicePlugin dp,
+ DeviceMappingManager dmm) {
+ deviceMappingManager = dmm;
resourceName = name;
devicePlugin = dp;
}
+ public DeviceMappingManager getDeviceMappingManager() {
+ return deviceMappingManager;
+ }
+
@Override
public void initialize(Context context) throws YarnException {
deviceResourceUpdater = new DeviceResourceUpdaterImpl(
@@ -62,7 +70,10 @@ public class DevicePluginAdapter implements ResourcePlugin {
public ResourceHandler createResourceHandler(Context nmContext,
CGroupsHandler cGroupsHandler,
PrivilegedOperationExecutor privilegedOperationExecutor) {
- return null;
+ this.deviceResourceHandler = new DeviceResourceHandlerImpl(resourceName,
+ devicePlugin, this, deviceMappingManager,
+ cGroupsHandler, privilegedOperationExecutor);
+ return deviceResourceHandler;
}
@Override
@@ -85,4 +96,7 @@ public class DevicePluginAdapter implements ResourcePlugin {
return null;
}
+ public DeviceResourceHandlerImpl getDeviceResourceHandler() {
+ return deviceResourceHandler;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java
new file mode 100644
index 0000000..d33b8da
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * The Hooks into container lifecycle.
+ * Get device list from device plugin in {@code bootstrap}
+ * Assign devices for a container in {@code preStart}
+ * Restore statue in {@code reacquireContainer}
+ * Recycle devices from container in {@code postComplete}
+ * */
+public class DeviceResourceHandlerImpl implements ResourceHandler {
+
+ static final Log LOG = LogFactory.getLog(DeviceResourceHandlerImpl.class);
+
+ private String resourceName;
+ private DevicePlugin devicePlugin;
+ private DeviceMappingManager deviceMappingManager;
+ private CGroupsHandler cGroupsHandler;
+ private PrivilegedOperationExecutor privilegedOperationExecutor;
+ private DevicePluginAdapter devicePluginAdapter;
+
+ public DeviceResourceHandlerImpl(String reseName,
+ DevicePlugin devPlugin,
+ DevicePluginAdapter devPluginAdapter,
+ DeviceMappingManager devMappingManager,
+ CGroupsHandler cgHandler,
+ PrivilegedOperationExecutor operation) {
+ this.devicePluginAdapter = devPluginAdapter;
+ this.resourceName = reseName;
+ this.devicePlugin = devPlugin;
+ this.cGroupsHandler = cgHandler;
+ this.privilegedOperationExecutor = operation;
+ this.deviceMappingManager = devMappingManager;
+ }
+
+ @Override
+ public List<PrivilegedOperation> bootstrap(Configuration configuration)
+ throws ResourceHandlerException {
+ Set<Device> availableDevices = null;
+ try {
+ availableDevices = devicePlugin.getDevices();
+ } catch (Exception e) {
+ throw new ResourceHandlerException("Exception thrown from"
+ + " plugin's \"getDevices\"" + e.getMessage());
+ }
+ /**
+ * We won't fail the NM if plugin returns invalid value here.
+ * */
+ if (availableDevices == null) {
+ LOG.error("Bootstrap " + resourceName
+ + " failed. Null value got from plugin's getDevices method");
+ return null;
+ }
+ // Add device set. Here we trust the plugin's return value
+ deviceMappingManager.addDeviceSet(resourceName, availableDevices);
+ // TODO: Init cgroups
+
+ return null;
+ }
+
+ @Override
+ public synchronized List<PrivilegedOperation> preStart(Container container)
+ throws ResourceHandlerException {
+ String containerIdStr = container.getContainerId().toString();
+ DeviceMappingManager.DeviceAllocation allocation =
+ deviceMappingManager.assignDevices(resourceName, container);
+ LOG.debug("Allocated to "
+ + containerIdStr + ": " + allocation);
+
+ try {
+ devicePlugin.onDevicesAllocated(
+ allocation.getAllowed(), YarnRuntimeType.RUNTIME_DEFAULT);
+ } catch (Exception e) {
+ throw new ResourceHandlerException("Exception thrown from"
+ + " plugin's \"onDeviceAllocated\"" + e.getMessage());
+ }
+
+ // cgroups operation based on allocation
+ /**
+ * TODO: implement a general container-executor device module
+ * */
+
+ return null;
+ }
+
+ @Override
+ public synchronized List<PrivilegedOperation> reacquireContainer(
+ ContainerId containerId) throws ResourceHandlerException {
+ deviceMappingManager.recoverAssignedDevices(resourceName, containerId);
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> updateContainer(Container container)
+ throws ResourceHandlerException {
+ return null;
+ }
+
+ @Override
+ public synchronized List<PrivilegedOperation> postComplete(
+ ContainerId containerId) throws ResourceHandlerException {
+ deviceMappingManager.cleanupAssignedDevices(resourceName, containerId);
+ return null;
+ }
+
+ @Override
+ public List<PrivilegedOperation> teardown()
+ throws ResourceHandlerException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDeviceMappingManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDeviceMappingManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDeviceMappingManager.java
new file mode 100644
index 0000000..d69ab42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDeviceMappingManager.java
@@ -0,0 +1,366 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.mockito.Mockito.*;
+
+/**
+ * Tests for DeviceMappingManager.
+ * Note that we test it under multi-threaded situation
+ * */
+public class TestDeviceMappingManager {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestDeviceMappingManager.class);
+
+ private String tempResourceTypesFile;
+ private DeviceMappingManager dmm;
+ private ExecutorService containerLauncher;
+ private Configuration conf;
+
+ @Before
+ public void setup() throws Exception {
+ // setup resource-types.xml
+ conf = new YarnConfiguration();
+ ResourceUtils.resetResourceTypes();
+ String resourceTypesFile = "resource-types-pluggable-devices.xml";
+ this.tempResourceTypesFile =
+ TestResourceUtils.setupResourceTypes(this.conf, resourceTypesFile);
+ NodeManager.NMContext context = mock(NodeManager.NMContext.class);
+ NMStateStoreService storeService = mock(NMStateStoreService.class);
+ when(context.getNMStateStore()).thenReturn(storeService);
+ doNothing().when(storeService).storeAssignedResources(isA(Container.class),
+ isA(String.class),
+ isA(ArrayList.class));
+ dmm = new DeviceMappingManager(context);
+ int deviceCount = 600;
+ TreeSet<Device> r = new TreeSet<>();
+ for (int i = 0; i < deviceCount; i++) {
+ r.add(Device.Builder.newInstance()
+ .setId(i)
+ .setDevPath("/dev/hdwA" + i)
+ .setMajorNumber(243)
+ .setMinorNumber(i)
+ .setBusID("0000:65:00." + i)
+ .setHealthy(true)
+ .build());
+ }
+ TreeSet<Device> r1 = new TreeSet<>();
+ for (int i = 0; i < deviceCount; i++) {
+ r1.add(Device.Builder.newInstance()
+ .setId(i)
+ .setDevPath("/dev/cmp" + i)
+ .setMajorNumber(100)
+ .setMinorNumber(i)
+ .setBusID("0000:11:00." + i)
+ .setHealthy(true)
+ .build());
+ }
+ dmm.addDeviceSet("cmpA.com/hdwA", r);
+ dmm.addDeviceSet("cmp.com/cmp", r1);
+
+ containerLauncher =
+ Executors.newFixedThreadPool(10);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ // cleanup resource-types.xml
+ File dest = new File(this.tempResourceTypesFile);
+ if (dest.exists()) {
+ boolean flag = dest.delete();
+ }
+ }
+
+ /**
+ * Simulate launch different containers requesting different resource.
+ * */
+ @Test
+ public void testAllocation()
+ throws InterruptedException, ResourceHandlerException {
+ int totalContainerCount = 100;
+ String resourceName1 = "cmpA.com/hdwA";
+ String resourceName2 = "cmp.com/cmp";
+ DeviceMappingManager dmmSpy = spy(dmm);
+ // generate a list of container
+ Map<String, Map<Container, Integer>> containerSet = new HashMap<>();
+ containerSet.put(resourceName1, new HashMap<>());
+ containerSet.put(resourceName2, new HashMap<>());
+ Long startTime = System.currentTimeMillis();
+ for (int i = 0; i < totalContainerCount; i++) {
+ // Random requeted device
+ int num = new Random().nextInt(5) + 1;
+ // Random requested resource type
+ String resourceName;
+ int seed = new Random().nextInt(5);
+ if (seed % 2 == 0) {
+ resourceName = resourceName1;
+ } else {
+ resourceName = resourceName2;
+ }
+ Container c = mockContainerWithDeviceRequest(i,
+ resourceName,
+ num, false);
+ containerSet.get(resourceName).put(c, num);
+
+ DeviceResourceHandlerImpl dri = new DeviceResourceHandlerImpl(
+ resourceName,
+ new MyTestPlugin(), null,
+ dmmSpy, null, null);
+ Future<Integer> f = containerLauncher.submit(new MyContainerLaunch(
+ dri, c, i, false));
+ }
+
+ containerLauncher.shutdown();
+ while (!containerLauncher.awaitTermination(10, TimeUnit.SECONDS)) {
+ LOG.info("Wait for the threads to finish");
+ }
+
+ Long endTime = System.currentTimeMillis();
+ LOG.info("Each container allocation spends roughly: {} ms",
+ (endTime - startTime)/totalContainerCount);
+ // Ensure invocation times
+ verify(dmmSpy, times(totalContainerCount)).assignDevices(
+ anyString(), any(Container.class));
+
+ // Ensure used devices' count for each type is correct
+ int totalAllocatedCount = 0;
+ Map<Device, ContainerId> used1 =
+ dmm.getAllUsedDevices().get(resourceName1);
+ Map<Device, ContainerId> used2 =
+ dmm.getAllUsedDevices().get(resourceName2);
+ for (Map.Entry<Container, Integer> entry :
+ containerSet.get(resourceName1).entrySet()) {
+ totalAllocatedCount += entry.getValue();
+ }
+ for (Map.Entry<Container, Integer> entry :
+ containerSet.get(resourceName2).entrySet()) {
+ totalAllocatedCount += entry.getValue();
+ }
+ Assert.assertEquals(totalAllocatedCount, used1.size() + used2.size());
+ // Ensure each container has correct devices
+ for (Map.Entry<Container, Integer> entry :
+ containerSet.get(resourceName1).entrySet()) {
+ int containerWanted = entry.getValue();
+ int actualAllocated = 0;
+ for (ContainerId cid : used1.values()) {
+ if (cid.equals(entry.getKey().getContainerId())) {
+ actualAllocated++;
+ }
+ }
+ Assert.assertEquals(containerWanted, actualAllocated);
+ }
+ for (Map.Entry<Container, Integer> entry :
+ containerSet.get(resourceName2).entrySet()) {
+ int containerWanted = entry.getValue();
+ int actualAllocated = 0;
+ for (ContainerId cid : used2.values()) {
+ if (cid.equals(entry.getKey().getContainerId())) {
+ actualAllocated++;
+ }
+ }
+ Assert.assertEquals(containerWanted, actualAllocated);
+ }
+ }
+
+ /**
+ * Simulate launch containers and cleanup.
+ * */
+ @Test
+ public void testAllocationAndCleanup()
+ throws InterruptedException, ResourceHandlerException, IOException {
+ int totalContainerCount = 10;
+ String resourceName1 = "cmpA.com/hdwA";
+ String resourceName2 = "cmp.com/cmp";
+ DeviceMappingManager dmmSpy = spy(dmm);
+ // generate a list of container
+ Map<String, Map<Container, Integer>> containerSet = new HashMap<>();
+ containerSet.put(resourceName1, new HashMap<>());
+ containerSet.put(resourceName2, new HashMap<>());
+ for (int i = 0; i < totalContainerCount; i++) {
+ // Random requeted device
+ int num = new Random().nextInt(5) + 1;
+ // Random requested resource type
+ String resourceName;
+ int seed = new Random().nextInt(5);
+ if (seed % 2 == 0) {
+ resourceName = resourceName1;
+ } else {
+ resourceName = resourceName2;
+ }
+ Container c = mockContainerWithDeviceRequest(i,
+ resourceName,
+ num, false);
+ containerSet.get(resourceName).put(c, num);
+
+ DeviceResourceHandlerImpl dri = new DeviceResourceHandlerImpl(
+ resourceName,
+ new MyTestPlugin(), null,
+ dmmSpy, null, null);
+ Future<Integer> f = containerLauncher.submit(new MyContainerLaunch(
+ dri, c, i, true));
+ }
+
+ containerLauncher.shutdown();
+ while (!containerLauncher.awaitTermination(10, TimeUnit.SECONDS)) {
+ LOG.info("Wait for the threads to finish");
+ }
+
+
+ // Ensure invocation times
+ verify(dmmSpy, times(totalContainerCount)).assignDevices(
+ anyString(), any(Container.class));
+ verify(dmmSpy, times(totalContainerCount)).cleanupAssignedDevices(
+ anyString(), any(ContainerId.class));
+
+ // Ensure all devices are back
+ Assert.assertEquals(0,
+ dmm.getAllUsedDevices().get(resourceName1).size());
+ Assert.assertEquals(0,
+ dmm.getAllUsedDevices().get(resourceName2).size());
+ }
+
+ private static Container mockContainerWithDeviceRequest(int id,
+ String resourceName,
+ int numDeviceRequest,
+ boolean dockerContainerEnabled) {
+ Container c = mock(Container.class);
+ when(c.getContainerId()).thenReturn(getContainerId(id));
+
+ Resource res = Resource.newInstance(1024, 1);
+ ResourceMappings resMapping = new ResourceMappings();
+
+ res.setResourceValue(resourceName, numDeviceRequest);
+ when(c.getResource()).thenReturn(res);
+ when(c.getResourceMappings()).thenReturn(resMapping);
+
+ ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+ Map<String, String> env = new HashMap<>();
+ if (dockerContainerEnabled) {
+ env.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE,
+ ContainerRuntimeConstants.CONTAINER_RUNTIME_DOCKER);
+ }
+ when(clc.getEnvironment()).thenReturn(env);
+ when(c.getLaunchContext()).thenReturn(clc);
+ return c;
+ }
+
+ private static ContainerId getContainerId(int id) {
+ return ContainerId.newContainerId(ApplicationAttemptId
+ .newInstance(ApplicationId.newInstance(1234L, 1), 1), id);
+ }
+
+ private static class MyContainerLaunch implements Callable<Integer> {
+ private DeviceResourceHandlerImpl deviceResourceHandler;
+ private Container container;
+ private boolean doCleanup;
+ private int cId;
+
+ MyContainerLaunch(DeviceResourceHandlerImpl dri,
+ Container c, int id, boolean cleanup) {
+ deviceResourceHandler = dri;
+ container = c;
+ doCleanup = cleanup;
+ cId = id;
+ }
+ @Override
+ public Integer call() throws Exception {
+ try {
+
+ deviceResourceHandler.preStart(container);
+ if (doCleanup) {
+ int seconds = new Random().nextInt(5);
+ LOG.info("sleep " + seconds);
+ Thread.sleep(seconds * 1000);
+ deviceResourceHandler.postComplete(getContainerId(cId));
+ }
+ } catch (ResourceHandlerException e) {
+ e.printStackTrace();
+ }
+ return 0;
+ }
+ }
+
+ private static class MyTestPlugin implements DevicePlugin {
+ private final static String RESOURCE_NAME = "abc";
+ @Override
+ public DeviceRegisterRequest getRegisterRequestInfo() {
+ return DeviceRegisterRequest.Builder.newInstance()
+ .setResourceName(RESOURCE_NAME).build();
+ }
+
+ @Override
+ public Set<Device> getDevices() {
+ TreeSet<Device> r = new TreeSet<>();
+ return r;
+ }
+
+ @Override
+ public DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
+ YarnRuntimeType yarnRuntime) throws Exception {
+ return null;
+ }
+
+ @Override
+ public void onDevicesReleased(Set<Device> releasedDevices) {
+
+ }
+ } // MyPlugin
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java
index c938b83..2534a0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java
@@ -18,14 +18,35 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
-import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.service.ServiceOperations;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.nodemanager.*;
-import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@@ -33,13 +54,23 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
/**
* Unit tests for DevicePluginAdapter.
@@ -52,6 +83,9 @@ public class TestDevicePluginAdapter {
private YarnConfiguration conf;
private String tempResourceTypesFile;
+ private CGroupsHandler mockCGroupsHandler;
+ private PrivilegedOperationExecutor mockPrivilegedExecutor;
+ private NodeManager nm;
@Before
public void setup() throws Exception {
@@ -61,6 +95,8 @@ public class TestDevicePluginAdapter {
String resourceTypesFile = "resource-types-pluggable-devices.xml";
this.tempResourceTypesFile =
TestResourceUtils.setupResourceTypes(this.conf, resourceTypesFile);
+ mockCGroupsHandler = mock(CGroupsHandler.class);
+ mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class);
}
@After
@@ -70,19 +106,132 @@ public class TestDevicePluginAdapter {
if (dest.exists()) {
dest.delete();
}
+ if (nm != null) {
+ try {
+ ServiceOperations.stop(nm);
+ } catch (Throwable t) {
+ // ignore
+ }
+ }
+ }
+
+
+ /**
+ * Use the MyPlugin which implement {@code DevicePlugin}.
+ * Plugin's initialization is tested in TestResourcePluginManager
+ * */
+ @Test
+ public void testBasicWorkflow()
+ throws YarnException, IOException {
+ NodeManager.NMContext context = mock(NodeManager.NMContext.class);
+ NMStateStoreService storeService = mock(NMStateStoreService.class);
+ when(context.getNMStateStore()).thenReturn(storeService);
+ doNothing().when(storeService).storeAssignedResources(isA(Container.class),
+ isA(String.class),
+ isA(ArrayList.class));
+
+ // Init scheduler manager
+ DeviceMappingManager dmm = new DeviceMappingManager(context);
+
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ when(rpm.getDeviceMappingManager()).thenReturn(dmm);
+
+ // Init an plugin
+ MyPlugin plugin = new MyPlugin();
+ MyPlugin spyPlugin = spy(plugin);
+ String resourceName = MyPlugin.RESOURCE_NAME;
+ // Init an adapter for the plugin
+ DevicePluginAdapter adapter = new DevicePluginAdapter(
+ resourceName,
+ spyPlugin, dmm);
+ // Bootstrap, adding device
+ adapter.initialize(context);
+ adapter.createResourceHandler(context,
+ mockCGroupsHandler, mockPrivilegedExecutor);
+ adapter.getDeviceResourceHandler().bootstrap(conf);
+ int size = dmm.getAvailableDevices(resourceName);
+ Assert.assertEquals(3, size);
+
+ // A container c1 requests 1 device
+ Container c1 = mockContainerWithDeviceRequest(0,
+ resourceName,
+ 1, false);
+ // preStart
+ adapter.getDeviceResourceHandler().preStart(c1);
+ // check book keeping
+ Assert.assertEquals(2,
+ dmm.getAvailableDevices(resourceName));
+ Assert.assertEquals(1,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+ // postComplete
+ adapter.getDeviceResourceHandler().postComplete(getContainerId(0));
+ Assert.assertEquals(3,
+ dmm.getAvailableDevices(resourceName));
+ Assert.assertEquals(0,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+
+ // A container c2 requests 3 device
+ Container c2 = mockContainerWithDeviceRequest(1,
+ resourceName,
+ 3, false);
+ // preStart
+ adapter.getDeviceResourceHandler().preStart(c2);
+ // check book keeping
+ Assert.assertEquals(0,
+ dmm.getAvailableDevices(resourceName));
+ Assert.assertEquals(3,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+ // postComplete
+ adapter.getDeviceResourceHandler().postComplete(getContainerId(1));
+ Assert.assertEquals(3,
+ dmm.getAvailableDevices(resourceName));
+ Assert.assertEquals(0,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+
+ // A container c3 request 0 device
+ Container c3 = mockContainerWithDeviceRequest(1,
+ resourceName,
+ 0, false);
+ // preStart
+ adapter.getDeviceResourceHandler().preStart(c3);
+ // check book keeping
+ Assert.assertEquals(3,
+ dmm.getAvailableDevices(resourceName));
+ Assert.assertEquals(0,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+ // postComplete
+ adapter.getDeviceResourceHandler().postComplete(getContainerId(1));
+ Assert.assertEquals(3,
+ dmm.getAvailableDevices(resourceName));
+ Assert.assertEquals(0,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
}
@Test
public void testDeviceResourceUpdaterImpl() throws YarnException {
Resource nodeResource = mock(Resource.class);
+ NodeManager.NMContext context = mock(NodeManager.NMContext.class);
// Init an plugin
MyPlugin plugin = new MyPlugin();
MyPlugin spyPlugin = spy(plugin);
String resourceName = MyPlugin.RESOURCE_NAME;
+ // Init scheduler manager
+ DeviceMappingManager dmm = new DeviceMappingManager(context);
// Init an adapter for the plugin
DevicePluginAdapter adapter = new DevicePluginAdapter(
- resourceName,
- spyPlugin);
+ resourceName, spyPlugin, dmm);
adapter.initialize(mock(Context.class));
adapter.getNodeResourceHandlerInstance()
.updateConfiguredResource(nodeResource);
@@ -91,6 +240,235 @@ public class TestDevicePluginAdapter {
resourceName, 3);
}
+ @Test
+ public void testStoreDeviceSchedulerManagerState()
+ throws IOException, YarnException {
+ NodeManager.NMContext context = mock(NodeManager.NMContext.class);
+ NMStateStoreService realStoreService = new NMMemoryStateStoreService();
+ NMStateStoreService storeService = spy(realStoreService);
+ when(context.getNMStateStore()).thenReturn(storeService);
+ doNothing().when(storeService).storeAssignedResources(isA(Container.class),
+ isA(String.class),
+ isA(ArrayList.class));
+
+ // Init scheduler manager
+ DeviceMappingManager dmm = new DeviceMappingManager(context);
+
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ when(rpm.getDeviceMappingManager()).thenReturn(dmm);
+
+ // Init an plugin
+ MyPlugin plugin = new MyPlugin();
+ MyPlugin spyPlugin = spy(plugin);
+ String resourceName = MyPlugin.RESOURCE_NAME;
+ // Init an adapter for the plugin
+ DevicePluginAdapter adapter = new DevicePluginAdapter(
+ resourceName,
+ spyPlugin, dmm);
+ // Bootstrap, adding device
+ adapter.initialize(context);
+ adapter.createResourceHandler(context,
+ mockCGroupsHandler, mockPrivilegedExecutor);
+ adapter.getDeviceResourceHandler().bootstrap(conf);
+
+ // A container c0 requests 1 device
+ Container c0 = mockContainerWithDeviceRequest(0,
+ resourceName,
+ 1, false);
+ // preStart
+ adapter.getDeviceResourceHandler().preStart(c0);
+ // ensure container1's resource is persistent
+ verify(storeService).storeAssignedResources(c0, resourceName,
+ Arrays.asList(Device.Builder.newInstance()
+ .setId(0)
+ .setDevPath("/dev/hdwA0")
+ .setMajorNumber(256)
+ .setMinorNumber(0)
+ .setBusID("0000:80:00.0")
+ .setHealthy(true)
+ .build()));
+ }
+
+ @Test
+ public void testRecoverDeviceSchedulerManagerState()
+ throws IOException, YarnException {
+ NodeManager.NMContext context = mock(NodeManager.NMContext.class);
+ NMStateStoreService realStoreService = new NMMemoryStateStoreService();
+ NMStateStoreService storeService = spy(realStoreService);
+ when(context.getNMStateStore()).thenReturn(storeService);
+ doNothing().when(storeService).storeAssignedResources(isA(Container.class),
+ isA(String.class),
+ isA(ArrayList.class));
+
+ // Init scheduler manager
+ DeviceMappingManager dmm = new DeviceMappingManager(context);
+
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ when(rpm.getDeviceMappingManager()).thenReturn(dmm);
+
+ // Init an plugin
+ MyPlugin plugin = new MyPlugin();
+ MyPlugin spyPlugin = spy(plugin);
+ String resourceName = MyPlugin.RESOURCE_NAME;
+ // Init an adapter for the plugin
+ DevicePluginAdapter adapter = new DevicePluginAdapter(
+ resourceName,
+ spyPlugin, dmm);
+ // Bootstrap, adding device
+ adapter.initialize(context);
+ adapter.createResourceHandler(context,
+ mockCGroupsHandler, mockPrivilegedExecutor);
+ adapter.getDeviceResourceHandler().bootstrap(conf);
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+ // mock NMStateStore
+ Device storedDevice = Device.Builder.newInstance()
+ .setId(0)
+ .setDevPath("/dev/hdwA0")
+ .setMajorNumber(256)
+ .setMinorNumber(0)
+ .setBusID("0000:80:00.0")
+ .setHealthy(true)
+ .build();
+ ConcurrentHashMap<ContainerId, Container> runningContainersMap
+ = new ConcurrentHashMap<>();
+ Container nmContainer = mock(Container.class);
+ ResourceMappings rmap = new ResourceMappings();
+ ResourceMappings.AssignedResources ar =
+ new ResourceMappings.AssignedResources();
+ ar.updateAssignedResources(
+ Arrays.asList(storedDevice));
+ rmap.addAssignedResources(resourceName, ar);
+ when(nmContainer.getResourceMappings()).thenReturn(rmap);
+ when(context.getContainers()).thenReturn(runningContainersMap);
+
+ // Test case 1. c0 get recovered. scheduler state restored
+ runningContainersMap.put(getContainerId(0), nmContainer);
+ adapter.getDeviceResourceHandler().reacquireContainer(
+ getContainerId(0));
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+ Assert.assertEquals(1,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(2,
+ dmm.getAvailableDevices(resourceName));
+ Map<Device, ContainerId> used = dmm.getAllUsedDevices().get(resourceName);
+ Assert.assertTrue(used.keySet().contains(storedDevice));
+
+ // Test case 2. c1 wants get recovered.
+ // But stored device is already allocated to c2
+ nmContainer = mock(Container.class);
+ rmap = new ResourceMappings();
+ ar = new ResourceMappings.AssignedResources();
+ ar.updateAssignedResources(
+ Arrays.asList(storedDevice));
+ rmap.addAssignedResources(resourceName, ar);
+ // already assigned to c1
+ runningContainersMap.put(getContainerId(2), nmContainer);
+ boolean caughtException = false;
+ try {
+ adapter.getDeviceResourceHandler().reacquireContainer(getContainerId(1));
+ } catch (ResourceHandlerException e) {
+ caughtException = true;
+ }
+ Assert.assertTrue(
+ "Should fail since requested device is assigned already",
+ caughtException);
+ // don't affect c0 allocation state
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+ Assert.assertEquals(1,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(2,
+ dmm.getAvailableDevices(resourceName));
+ used = dmm.getAllUsedDevices().get(resourceName);
+ Assert.assertTrue(used.keySet().contains(storedDevice));
+ }
+
+ @Test
+ public void testAssignedDeviceCleanupWhenStoreOpFails()
+ throws IOException, YarnException {
+ NodeManager.NMContext context = mock(NodeManager.NMContext.class);
+ NMStateStoreService realStoreService = new NMMemoryStateStoreService();
+ NMStateStoreService storeService = spy(realStoreService);
+ when(context.getNMStateStore()).thenReturn(storeService);
+ doThrow(new IOException("Exception ...")).when(storeService)
+ .storeAssignedResources(isA(Container.class),
+ isA(String.class),
+ isA(ArrayList.class));
+
+ // Init scheduler manager
+ DeviceMappingManager dmm = new DeviceMappingManager(context);
+
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ when(rpm.getDeviceMappingManager()).thenReturn(dmm);
+
+ // Init an plugin
+ MyPlugin plugin = new MyPlugin();
+ MyPlugin spyPlugin = spy(plugin);
+ String resourceName = MyPlugin.RESOURCE_NAME;
+ // Init an adapter for the plugin
+ DevicePluginAdapter adapter = new DevicePluginAdapter(
+ resourceName,
+ spyPlugin, dmm);
+ // Bootstrap, adding device
+ adapter.initialize(context);
+ adapter.createResourceHandler(context,
+ mockCGroupsHandler, mockPrivilegedExecutor);
+ adapter.getDeviceResourceHandler().bootstrap(conf);
+
+ // A container c0 requests 1 device
+ Container c0 = mockContainerWithDeviceRequest(0,
+ resourceName,
+ 1, false);
+ // preStart
+ boolean exception = false;
+ try {
+ adapter.getDeviceResourceHandler().preStart(c0);
+ } catch (ResourceHandlerException e) {
+ exception = true;
+ }
+ Assert.assertTrue("Should throw exception in preStart", exception);
+ // no device assigned
+ Assert.assertEquals(3,
+ dmm.getAllAllowedDevices().get(resourceName).size());
+ Assert.assertEquals(0,
+ dmm.getAllUsedDevices().get(resourceName).size());
+ Assert.assertEquals(3,
+ dmm.getAvailableDevices(resourceName));
+
+ }
+
+ private static Container mockContainerWithDeviceRequest(int id,
+ String resourceName,
+ int numDeviceRequest,
+ boolean dockerContainerEnabled) {
+ Container c = mock(Container.class);
+ when(c.getContainerId()).thenReturn(getContainerId(id));
+
+ Resource res = Resource.newInstance(1024, 1);
+ ResourceMappings resMapping = new ResourceMappings();
+
+ res.setResourceValue(resourceName, numDeviceRequest);
+ when(c.getResource()).thenReturn(res);
+ when(c.getResourceMappings()).thenReturn(resMapping);
+
+ ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
+ Map<String, String> env = new HashMap<>();
+ if (dockerContainerEnabled) {
+ env.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE,
+ ContainerRuntimeConstants.CONTAINER_RUNTIME_DOCKER);
+ }
+ when(clc.getEnvironment()).thenReturn(env);
+ when(c.getLaunchContext()).thenReturn(clc);
+ return c;
+ }
+
+ private static ContainerId getContainerId(int id) {
+ return ContainerId.newContainerId(ApplicationAttemptId
+ .newInstance(ApplicationId.newInstance(1234L, 1), 1), id);
+ }
+
private class MyPlugin implements DevicePlugin {
private final static String RESOURCE_NAME = "cmpA.com/hdwA";
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-9041. Performance
Optimization of method FSPreemptionThread#identifyContainersToPreempt.
Contributed by Wanqiang Ji.
Posted by sh...@apache.org.
YARN-9041. Performance Optimization of method FSPreemptionThread#identifyContainersToPreempt. Contributed by Wanqiang Ji.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e89941fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e89941fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e89941fd
Branch: refs/heads/HDFS-12943
Commit: e89941fdbb3b382eeb487d32e5194909610ac334
Parents: db2d8b0
Author: Yufei Gu <yu...@apache.org>
Authored: Tue Dec 4 10:08:45 2018 -0800
Committer: Yufei Gu <yu...@apache.org>
Committed: Tue Dec 4 10:08:45 2018 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSPreemptionThread.java | 55 ++++---
.../fair/TestFairSchedulerPreemption.java | 155 ++++++++++++-------
2 files changed, 140 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e89941fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index 6ed90f8..4c83052 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -97,12 +97,7 @@ class FSPreemptionThread extends Thread {
* Mechanics:
* 1. Fetch all {@link ResourceRequest}s corresponding to the amount of
* starvation.
- * 2. For each {@link ResourceRequest}, iterate through matching
- * nodes and identify containers to preempt all on one node, also
- * optimizing for least number of AM container preemptions. Only nodes
- * that match the locality level specified in the {@link ResourceRequest}
- * are considered. However, if this would lead to AM preemption, and locality
- * relaxation is allowed, then the search space is expanded to all nodes.
+ * 2. For each {@link ResourceRequest}, get the best preemptable containers.
*
* @param starvedApp starved application for which we are identifying
* preemption targets
@@ -118,18 +113,7 @@ class FSPreemptionThread extends Thread {
.getNodesByResourceName(rr.getResourceName());
for (int i = 0; i < rr.getNumContainers(); i++) {
PreemptableContainers bestContainers =
- identifyContainersToPreemptForOneContainer(potentialNodes, rr);
-
- // Don't preempt AM containers just to satisfy local requests if relax
- // locality is enabled.
- if (bestContainers != null
- && bestContainers.numAMContainers > 0
- && !ResourceRequest.isAnyLocation(rr.getResourceName())
- && rr.getRelaxLocality()) {
- bestContainers = identifyContainersToPreemptForOneContainer(
- scheduler.getNodeTracker().getAllNodes(), rr);
- }
-
+ getBestPreemptableContainers(rr, potentialNodes);
if (bestContainers != null) {
List<RMContainer> containers = bestContainers.getAllContainers();
if (containers.size() > 0) {
@@ -240,6 +224,41 @@ class FSPreemptionThread extends Thread {
new PreemptContainersTask(containers), warnTimeBeforeKill);
}
+ /**
+ * Iterate through matching nodes and identify containers to preempt all on
+ * one node, also optimizing for least number of AM container preemptions.
+ * Only nodes that match the locality level specified in the
+ * {@link ResourceRequest} are considered. However, if this would lead to
+ * AM preemption, and locality relaxation is allowed, then the search space
+ * is expanded to the remaining nodes.
+ *
+ * @param rr resource request
+ * @param potentialNodes list of {@link FSSchedulerNode}
+ * @return the list of best preemptable containers for the resource request
+ */
+ private PreemptableContainers getBestPreemptableContainers(ResourceRequest rr,
+ List<FSSchedulerNode> potentialNodes) {
+ PreemptableContainers bestContainers =
+ identifyContainersToPreemptForOneContainer(potentialNodes, rr);
+
+ if (rr.getRelaxLocality()
+ && !ResourceRequest.isAnyLocation(rr.getResourceName())
+ && bestContainers != null
+ && bestContainers.numAMContainers > 0) {
+ List<FSSchedulerNode> remainingNodes =
+ scheduler.getNodeTracker().getAllNodes();
+ remainingNodes.removeAll(potentialNodes);
+ PreemptableContainers spareContainers =
+ identifyContainersToPreemptForOneContainer(remainingNodes, rr);
+ if (spareContainers != null && spareContainers.numAMContainers
+ < bestContainers.numAMContainers) {
+ bestContainers = spareContainers;
+ }
+ }
+
+ return bestContainers;
+ }
+
private class PreemptContainersTask extends TimerTask {
private final List<RMContainer> containers;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e89941fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index da6428a..67cb600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -387,13 +387,6 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
}
}
- private void setAllAMContainersOnNode(NodeId nodeId) {
- SchedulerNode node = scheduler.getNodeTracker().getNode(nodeId);
- for (RMContainer container: node.getCopiedListOfRunningContainers()) {
- ((RMContainerImpl) container).setAMContainer(true);
- }
- }
-
@Test
public void testPreemptionSelectNonAMContainer() throws Exception {
takeAllResources("root.preemptable.child-1");
@@ -413,51 +406,6 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
}
@Test
- public void testRelaxLocalityToNotPreemptAM() throws Exception {
- takeAllResources("root.preemptable.child-1");
- RMNode node1 = rmNodes.get(0);
- setAllAMContainersOnNode(node1.getNodeID());
- SchedulerNode node = scheduler.getNodeTracker().getNode(node1.getNodeID());
- ApplicationAttemptId greedyAppAttemptId =
- node.getCopiedListOfRunningContainers().get(0)
- .getApplicationAttemptId();
-
- // Make the RACK_LOCAL and OFF_SWITCH requests big enough that they can't be
- // satisfied. This forces the RR that we consider for preemption to be the
- // NODE_LOCAL one.
- ResourceRequest nodeRequest =
- createResourceRequest(GB, node1.getHostName(), 1, 4, true);
- ResourceRequest rackRequest =
- createResourceRequest(GB * 10, node1.getRackName(), 1, 1, true);
- ResourceRequest anyRequest =
- createResourceRequest(GB * 10, ResourceRequest.ANY, 1, 1, true);
-
- List<ResourceRequest> resourceRequests =
- Arrays.asList(nodeRequest, rackRequest, anyRequest);
-
- ApplicationAttemptId starvedAppAttemptId = createSchedulingRequest(
- "root.preemptable.child-2", "default", resourceRequests);
- starvingApp = scheduler.getSchedulerApp(starvedAppAttemptId);
-
- // Move clock enough to identify starvation
- clock.tickSec(1);
- scheduler.update();
-
- // Make sure 4 containers were preempted from the greedy app, but also that
- // none were preempted on our all-AM node, even though the NODE_LOCAL RR
- // asked for resources on it.
-
- // TODO (YARN-7655) The starved app should be allocated 4 containers.
- // It should be possible to modify the RRs such that this is true
- // after YARN-7903.
- verifyPreemption(0, 4);
- for (RMContainer container : node.getCopiedListOfRunningContainers()) {
- assert (container.isAMContainer());
- assert (container.getApplicationAttemptId().equals(greedyAppAttemptId));
- }
- }
-
- @Test
public void testAppNotPreemptedBelowFairShare() throws Exception {
takeAllResources("root.preemptable.child-1");
tryPreemptMoreThanFairShare("root.preemptable.child-2");
@@ -492,4 +440,107 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
preemptHalfResources("root.preemptable.child-2");
verifyPreemption(1, 2);
}
+
+ /* It tests the case that there is less-AM-container solution in the
+ * remaining nodes.
+ */
+ @Test
+ public void testRelaxLocalityPreemptionWithLessAMInRemainingNodes()
+ throws Exception {
+ takeAllResources("root.preemptable.child-1");
+ RMNode node1 = rmNodes.get(0);
+ setAllAMContainersOnNode(node1.getNodeID());
+ ApplicationAttemptId greedyAppAttemptId =
+ getGreedyAppAttemptIdOnNode(node1.getNodeID());
+ updateRelaxLocalityRequestSchedule(node1, GB, 4);
+ verifyRelaxLocalityPreemption(node1.getNodeID(), greedyAppAttemptId, 4);
+ }
+
+ /* It tests the case that there is no less-AM-container solution in the
+ * remaining nodes.
+ */
+ @Test
+ public void testRelaxLocalityPreemptionWithNoLessAMInRemainingNodes()
+ throws Exception {
+ takeAllResources("root.preemptable.child-1");
+ RMNode node1 = rmNodes.get(0);
+ setNumAMContainersOnNode(3, node1.getNodeID());
+ RMNode node2 = rmNodes.get(1);
+ setAllAMContainersOnNode(node2.getNodeID());
+ ApplicationAttemptId greedyAppAttemptId =
+ getGreedyAppAttemptIdOnNode(node2.getNodeID());
+ updateRelaxLocalityRequestSchedule(node1, GB * 2, 1);
+ verifyRelaxLocalityPreemption(node2.getNodeID(), greedyAppAttemptId, 6);
+ }
+
+ private void setAllAMContainersOnNode(NodeId nodeId) {
+ setNumAMContainersOnNode(Integer.MAX_VALUE, nodeId);
+ }
+
+ private void setNumAMContainersOnNode(int num, NodeId nodeId) {
+ int count = 0;
+ SchedulerNode node = scheduler.getNodeTracker().getNode(nodeId);
+ for (RMContainer container: node.getCopiedListOfRunningContainers()) {
+ count++;
+ if (count <= num) {
+ ((RMContainerImpl) container).setAMContainer(true);
+ } else {
+ break;
+ }
+ }
+ }
+
+ private ApplicationAttemptId getGreedyAppAttemptIdOnNode(NodeId nodeId) {
+ SchedulerNode node = scheduler.getNodeTracker().getNode(nodeId);
+ return node.getCopiedListOfRunningContainers().get(0)
+ .getApplicationAttemptId();
+ }
+
+ /*
+ * Send the resource requests allowed relax locality to scheduler. The
+ * params node/nodeMemory/numNodeContainers used for NODE_LOCAL request.
+ */
+ private void updateRelaxLocalityRequestSchedule(RMNode node, int nodeMemory,
+ int numNodeContainers) {
+ // Make the RACK_LOCAL and OFF_SWITCH requests big enough that they can't be
+ // satisfied. This forces the RR that we consider for preemption to be the
+ // NODE_LOCAL one.
+ ResourceRequest nodeRequest = createResourceRequest(nodeMemory,
+ node.getHostName(), 1, numNodeContainers, true);
+ ResourceRequest rackRequest =
+ createResourceRequest(GB * 10, node.getRackName(), 1, 1, true);
+ ResourceRequest anyRequest =
+ createResourceRequest(GB * 10, ResourceRequest.ANY, 1, 1, true);
+
+ List<ResourceRequest> resourceRequests =
+ Arrays.asList(nodeRequest, rackRequest, anyRequest);
+
+ ApplicationAttemptId starvedAppAttemptId = createSchedulingRequest(
+ "root.preemptable.child-2", "default", resourceRequests);
+ starvingApp = scheduler.getSchedulerApp(starvedAppAttemptId);
+
+ // Move clock enough to identify starvation
+ clock.tickSec(1);
+ scheduler.update();
+ }
+
+ private void verifyRelaxLocalityPreemption(NodeId notBePreemptedNodeId,
+ ApplicationAttemptId greedyAttemptId, int numGreedyAppContainers)
+ throws Exception {
+ // Make sure 4 containers were preempted from the greedy app, but also that
+ // none were preempted on our all-AM node, even though the NODE_LOCAL RR
+ // asked for resources on it.
+
+ // TODO (YARN-7655) The starved app should be allocated 4 containers.
+ // It should be possible to modify the RRs such that this is true
+ // after YARN-7903.
+ verifyPreemption(0, numGreedyAppContainers);
+ SchedulerNode node = scheduler.getNodeTracker()
+ .getNode(notBePreemptedNodeId);
+ for (RMContainer container : node.getCopiedListOfRunningContainers()) {
+ assert(container.isAMContainer());
+ assert(container.getApplicationAttemptId().equals(greedyAttemptId));
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDDS-642. Add chill mode exit
condition for pipeline availability. Contributed by Yiqun Lin.
Posted by sh...@apache.org.
HDDS-642. Add chill mode exit condition for pipeline availability. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b71cc7f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b71cc7f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b71cc7f3
Branch: refs/heads/HDFS-12943
Commit: b71cc7f33edbbf6a98d1efb330f1c748b5dd6e75
Parents: efc4d91
Author: Ajay Kumar <aj...@apache.org>
Authored: Wed Nov 28 17:45:46 2018 -0800
Committer: Ajay Kumar <aj...@apache.org>
Committed: Wed Nov 28 17:47:57 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +
.../common/src/main/resources/ozone-default.xml | 9 ++
.../scm/chillmode/PipelineChillModeRule.java | 108 +++++++++++++++++++
.../hdds/scm/chillmode/SCMChillModeManager.java | 19 +++-
.../scm/server/StorageContainerManager.java | 5 +-
.../scm/chillmode/TestSCMChillModeManager.java | 81 ++++++++++++--
6 files changed, 213 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 2d28a5b..f16503e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -87,6 +87,11 @@ public final class HddsConfigKeys {
"hdds.scm.chillmode.min.datanode";
public static final int HDDS_SCM_CHILLMODE_MIN_DATANODE_DEFAULT = 1;
+ public static final String HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK =
+ "hdds.scm.chillmode.pipeline-availability.check";
+ public static final boolean
+ HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT = false;
+
// % of containers which should have at least one reported replica
// before SCM comes out of chill mode.
public static final String HDDS_SCM_CHILLMODE_THRESHOLD_PCT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 9f3d7e1..aa22b2b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1232,6 +1232,15 @@
</property>
<property>
+ <name>hdds.scm.chillmode.pipeline-availability.check</name>
+ <value>false</value>
+ <tag>HDDS,SCM,OPERATION</tag>
+ <description>
+ Boolean value to enable pipeline availability check during SCM chill mode.
+ </description>
+ </property>
+
+ <property>
<name>hdds.container.action.max.limit</name>
<value>20</value>
<tag>DATANODE</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
new file mode 100644
index 0000000..f9a6e59
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/PipelineChillModeRule.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.chillmode;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Class defining Chill mode exit criteria for Pipelines.
+ */
+public class PipelineChillModeRule
+ implements ChillModeExitRule<PipelineReportFromDatanode>,
+ EventHandler<PipelineReportFromDatanode> {
+ /** Pipeline availability.*/
+ private AtomicBoolean isPipelineAvailable = new AtomicBoolean(false);
+
+ private final PipelineManager pipelineManager;
+ private final SCMChillModeManager chillModeManager;
+
+ PipelineChillModeRule(PipelineManager pipelineManager,
+ SCMChillModeManager manager) {
+ this.pipelineManager = pipelineManager;
+ this.chillModeManager = manager;
+ }
+
+ @Override
+ public boolean validate() {
+ return isPipelineAvailable.get();
+ }
+
+ @Override
+ public void process(PipelineReportFromDatanode report) {
+ // No need to deal with
+ }
+
+ @Override
+ public void cleanup() {
+ // No need to deal with
+ }
+
+ @Override
+ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode,
+ EventPublisher publisher) {
+ // If we are already in pipeline available state,
+ // skipping following check.
+ if (validate()) {
+ chillModeManager.validateChillModeExitRules(publisher);
+ return;
+ }
+
+ Pipeline pipeline;
+ Preconditions.checkNotNull(pipelineReportFromDatanode);
+ PipelineReportsProto pipelineReport = pipelineReportFromDatanode
+ .getReport();
+
+ for (PipelineReport report : pipelineReport.getPipelineReportList()) {
+ PipelineID pipelineID = PipelineID
+ .getFromProtobuf(report.getPipelineID());
+ try {
+ pipeline = pipelineManager.getPipeline(pipelineID);
+ } catch (PipelineNotFoundException e) {
+ continue;
+ }
+
+ if (pipeline.getPipelineState() == Pipeline.PipelineState.OPEN) {
+ // ensure there is an OPEN state pipeline and then allowed
+ // to exit chill mode
+ isPipelineAvailable.set(true);
+
+ if (chillModeManager.getInChillMode()) {
+ SCMChillModeManager.getLogger()
+ .info("SCM in chill mode. 1 Pipeline reported, 1 required.");
+ }
+ break;
+ }
+ }
+
+ if (validate()) {
+ chillModeManager.validateChillModeExitRules(publisher);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
index fa0eaa9..09b998d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/chillmode/SCMChillModeManager.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer
.NodeRegistrationContainerReport;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -53,14 +54,18 @@ public class SCMChillModeManager implements
LoggerFactory.getLogger(SCMChillModeManager.class);
private final boolean isChillModeEnabled;
private AtomicBoolean inChillMode = new AtomicBoolean(true);
+
private Map<String, ChillModeExitRule> exitRules = new HashMap(1);
private Configuration config;
private static final String CONT_EXIT_RULE = "ContainerChillModeRule";
private static final String DN_EXIT_RULE = "DataNodeChillModeRule";
+ private static final String PIPELINE_EXIT_RULE = "PipelineChillModeRule";
+
private final EventQueue eventPublisher;
public SCMChillModeManager(Configuration conf,
- List<ContainerInfo> allContainers, EventQueue eventQueue) {
+ List<ContainerInfo> allContainers, PipelineManager pipelineManager,
+ EventQueue eventQueue) {
this.config = conf;
this.eventPublisher = eventQueue;
this.isChillModeEnabled = conf.getBoolean(
@@ -70,6 +75,16 @@ public class SCMChillModeManager implements
exitRules.put(CONT_EXIT_RULE,
new ContainerChillModeRule(config, allContainers, this));
exitRules.put(DN_EXIT_RULE, new DataNodeChillModeRule(config, this));
+
+ if (conf.getBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT)
+ && pipelineManager != null) {
+ PipelineChillModeRule rule = new PipelineChillModeRule(pipelineManager,
+ this);
+ exitRules.put(PIPELINE_EXIT_RULE, rule);
+ eventPublisher.addHandler(SCMEvents.PIPELINE_REPORT, rule);
+ }
emitChillModeStatus();
} else {
exitChillMode(eventQueue);
@@ -84,7 +99,7 @@ public class SCMChillModeManager implements
eventPublisher.fireEvent(SCMEvents.CHILL_MODE_STATUS, getInChillMode());
}
- private void validateChillModeExitRules(EventPublisher eventQueue) {
+ public void validateChillModeExitRules(EventPublisher eventQueue) {
for (ChillModeExitRule exitRule : exitRules.values()) {
if (!exitRule.validate()) {
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index fe00b54..2d27984 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -217,7 +217,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
NodeReportHandler nodeReportHandler =
new NodeReportHandler(scmNodeManager);
PipelineReportHandler pipelineReportHandler =
- new PipelineReportHandler(pipelineManager, conf);
+ new PipelineReportHandler(pipelineManager, conf);
CommandStatusReportHandler cmdStatusReportHandler =
new CommandStatusReportHandler();
@@ -302,8 +302,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
(BlockManagerImpl) scmBlockManager);
scmChillModeManager = new SCMChillModeManager(conf,
- containerManager.getContainers(),
- eventQueue);
+ containerManager.getContainers(), pipelineManager, eventQueue);
eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b71cc7f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
index 0487fb7..ed17796 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/chillmode/TestSCMChillModeManager.java
@@ -17,15 +17,29 @@
*/
package org.apache.hadoop.hdds.scm.chillmode;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
import java.util.ArrayList;
import java.util.List;
+import java.util.UUID;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.BeforeClass;
@@ -33,9 +47,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
/** Test class for SCMChillModeManager.
*/
public class TestSCMChillModeManager {
@@ -65,7 +76,7 @@ public class TestSCMChillModeManager {
@Test
public void testChillModeStateWithNullContainers() {
- new SCMChillModeManager(config, null, queue);
+ new SCMChillModeManager(config, null, null, queue);
}
private void testChillMode(int numContainers) throws Exception {
@@ -76,7 +87,8 @@ public class TestSCMChillModeManager {
for (ContainerInfo container : containers) {
container.setState(HddsProtos.LifeCycleState.OPEN);
}
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
assertTrue(scmChillModeManager.getInChillMode());
@@ -96,7 +108,8 @@ public class TestSCMChillModeManager {
for (ContainerInfo container : containers) {
container.setState(HddsProtos.LifeCycleState.CLOSED);
}
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
assertTrue(scmChillModeManager.getInChillMode());
@@ -118,7 +131,8 @@ public class TestSCMChillModeManager {
public void testDisableChillMode() {
OzoneConfiguration conf = new OzoneConfiguration(config);
conf.setBoolean(HddsConfigKeys.HDDS_SCM_CHILLMODE_ENABLED, false);
- scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ conf, containers, null, queue);
assertFalse(scmChillModeManager.getInChillMode());
}
@@ -149,7 +163,8 @@ public class TestSCMChillModeManager {
container.setState(HddsProtos.LifeCycleState.OPEN);
}
- scmChillModeManager = new SCMChillModeManager(config, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
assertTrue(scmChillModeManager.getInChillMode());
@@ -173,7 +188,8 @@ public class TestSCMChillModeManager {
private void testChillModeDataNodes(int numOfDns) throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(config);
conf.setInt(HddsConfigKeys.HDDS_SCM_CHILLMODE_MIN_DATANODE, numOfDns);
- scmChillModeManager = new SCMChillModeManager(conf, containers, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ conf, containers, null, queue);
queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
scmChillModeManager);
// Assert SCM is in Chill mode.
@@ -212,4 +228,51 @@ public class TestSCMChillModeManager {
}, 100, 2000 * 9);
}
+ @Test
+ public void testChillModePipelineExitRule() throws Exception {
+ containers = new ArrayList<>();
+ containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
+ String storageDir = GenericTestUtils.getTempPath(
+ TestSCMChillModeManager.class.getName() + UUID.randomUUID());
+ try{
+ MockNodeManager nodeManager = new MockNodeManager(true, 1);
+ config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir);
+ // enable pipeline check
+ config.setBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK, true);
+
+ PipelineManager pipelineManager = new SCMPipelineManager(config,
+ nodeManager, queue);
+ scmChillModeManager = new SCMChillModeManager(
+ config, containers, pipelineManager, queue);
+ queue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ HddsTestUtils.createNodeRegistrationContainerReport(containers));
+ assertTrue(scmChillModeManager.getInChillMode());
+
+ // simulation a pipeline report to trigger the rule check
+ Pipeline pipeline = pipelineManager.createPipeline(
+ HddsProtos.ReplicationType.STAND_ALONE,
+ HddsProtos.ReplicationFactor.ONE);
+ PipelineReportsProto.Builder reportBuilder = PipelineReportsProto
+ .newBuilder();
+ reportBuilder.addPipelineReport(PipelineReport.newBuilder()
+ .setPipelineID(pipeline.getId().getProtobuf()));
+
+ queue.fireEvent(SCMEvents.PIPELINE_REPORT, new PipelineReportFromDatanode(
+ pipeline.getNodes().get(0), reportBuilder.build()));
+
+ GenericTestUtils.waitFor(() -> {
+ return !scmChillModeManager.getInChillMode();
+ }, 100, 1000 * 10);
+ pipelineManager.close();
+ } finally {
+ config.setBoolean(
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_PIPELINE_AVAILABILITY_CHECK,
+ false);
+ FileUtil.fullyDelete(new File(storageDir));
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDDS-885. Fix test failures due to
ChecksumData. Contributed by Hanisha Koneru.
Posted by sh...@apache.org.
HDDS-885. Fix test failures due to ChecksumData. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef3b03b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef3b03b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef3b03b7
Branch: refs/heads/HDFS-12943
Commit: ef3b03b75abd5ca5b96dab7985aa8fe17578fa07
Parents: fb10803
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Dec 3 10:52:49 2018 -0800
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Dec 3 10:52:49 2018 -0800
----------------------------------------------------------------------
.../hdds/scm/storage/ContainerProtocolCalls.java | 14 +++++++++++---
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +++--
.../org/apache/hadoop/ozone/common/Checksum.java | 14 ++++++++++++--
.../ozone/container/common/helpers/ChunkInfo.java | 9 ++++++++-
.../common/src/main/resources/ozone-default.xml | 18 ++++++++++++++++++
.../container/common/impl/TestHddsDispatcher.java | 2 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 5 +++--
.../common/TestBlockDeletingService.java | 4 ++--
.../container/common/helpers/TestBlockData.java | 7 ++++++-
.../web/storage/DistributedStorageHandler.java | 5 +++--
10 files changed, 68 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
index 1f20d00..8af3973 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.hdds.scm.storage;
import org.apache.hadoop.hdds.scm.XceiverClientAsyncReply;
import org.apache.hadoop.hdds.scm.container.common.helpers
.BlockNotCommittedException;
+import org.apache.hadoop.ozone.common.Checksum;
+import org.apache.hadoop.ozone.common.ChecksumData;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers
@@ -305,10 +307,16 @@ public final class ContainerProtocolCalls {
KeyValue keyValue =
KeyValue.newBuilder().setKey("OverWriteRequested").setValue("true")
.build();
+ Checksum checksum = new Checksum();
+ ChecksumData checksumData = checksum.computeChecksum(data);
ChunkInfo chunk =
- ChunkInfo.newBuilder().setChunkName(blockID.getLocalID()
- + "_chunk").setOffset(0).setLen(data.length).
- addMetadata(keyValue).build();
+ ChunkInfo.newBuilder()
+ .setChunkName(blockID.getLocalID() + "_chunk")
+ .setOffset(0)
+ .setLen(data.length)
+ .addMetadata(keyValue)
+ .setChecksumData(checksumData.getProtoBufMessage())
+ .build();
PutSmallFileRequestProto putSmallFileRequest =
PutSmallFileRequestProto.newBuilder().setChunkInfo(chunk)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 496861c..a331f48 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -342,8 +342,9 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_CHECKSUM_TYPE_DEFAULT = "SHA256";
public static final String OZONE_CLIENT_BYTES_PER_CHECKSUM =
"ozone.client.bytes.per.checksum";
- public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT =
- 1024 * 1024; // 1 MB
+ public static final String OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT = "1MB";
+ public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT_BYTES =
+ 1024 * 1024;
public static final int OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE = 256 * 1024;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
index 83293e5..56ad71b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Checksum.java
@@ -24,7 +24,9 @@ import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ChecksumType;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -67,7 +69,7 @@ public class Checksum {
this.checksumType = ChecksumType.valueOf(
OzoneConfigKeys.OZONE_CLIENT_CHECKSUM_TYPE_DEFAULT);
this.bytesPerChecksum = OzoneConfigKeys
- .OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT;
+ .OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT_BYTES; // Default is 1MB
}
/**
@@ -236,4 +238,12 @@ public class Checksum {
return checksumData.verifyChecksumDataMatches(computedChecksumData);
}
+
+ /**
+ * Returns a ChecksumData with type NONE for testing.
+ */
+ @VisibleForTesting
+ public static ContainerProtos.ChecksumData getNoChecksumDataProto() {
+ return new ChecksumData(ChecksumType.NONE, 0).getProtoBufMessage();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
index d75f10f..1c73a31 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
+import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChecksumData;
/**
@@ -105,7 +106,13 @@ public class ChunkInfo {
builder.setChunkName(this.getChunkName());
builder.setOffset(this.getOffset());
builder.setLen(this.getLen());
- builder.setChecksumData(this.checksumData.getProtoBufMessage());
+ if (checksumData == null) {
+ // ChecksumData cannot be null while computing the protobufMessage.
+ // Set it to NONE type (equivalent to non checksum).
+ builder.setChecksumData(Checksum.getNoChecksumDataProto());
+ } else {
+ builder.setChecksumData(this.checksumData.getProtoBufMessage());
+ }
for (Map.Entry<String, String> entry : metadata.entrySet()) {
ContainerProtos.KeyValue.Builder keyValBuilder =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index edce616..d8e6c78 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1397,5 +1397,23 @@
</description>
</property>
+ <property>
+ <name>ozone.client.checksum.type</name>
+ <value>SHA256</value>
+ <tag>OZONE, CLIENT, MANAGEMENT</tag>
+ <description>The checksum type [NONE/ CRC32/ CRC32C/ SHA256/ MD5] determines
+ which algorithm would be used to compute checksum for chunk data.
+ Default checksum type is SHA256.
+ </description>
+ </property>
+
+ <property>
+ <name>ozone.client.bytes.per.checksum</name>
+ <value>1MB</value>
+ <tag>OZONE, CLIENT, MANAGEMENT</tag>
+ <description>Checksum will be computed for every bytes per checksum number
+ of bytes and stored sequentially.
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index 933ed70..4727261 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.WriteChunkRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerAction;
+import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
@@ -199,6 +200,7 @@ public class TestHddsDispatcher {
+ containerId + "_chunk_" + localId)
.setOffset(0)
.setLen(data.size())
+ .setChecksumData(Checksum.getNoChecksumDataProto())
.build();
WriteChunkRequestProto.Builder writeChunkRequest = WriteChunkRequestProto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 65adbfa..334f713 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -171,9 +171,10 @@ public class RpcClient implements ClientProtocol {
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
- int configuredChecksumSize = conf.getInt(
+ int configuredChecksumSize = (int) conf.getStorageSize(
OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM,
- OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT);
+ OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT,
+ StorageUnit.BYTES);
int checksumSize;
if(configuredChecksumSize <
OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 0db047f..e8ae266 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -140,8 +141,7 @@ public class TestBlockDeletingService {
.setChunkName(chunk.getAbsolutePath())
.setLen(0)
.setOffset(0)
- .setChecksumData(
- ContainerProtos.ChecksumData.getDefaultInstance())
+ .setChecksumData(Checksum.getNoChecksumDataProto())
.build();
chunks.add(info);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java
index 9df4249..2973a76 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestBlockData.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.helpers;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.ozone.common.Checksum;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -41,7 +42,11 @@ public class TestBlockData {
static ContainerProtos.ChunkInfo buildChunkInfo(String name, long offset,
long len) {
return ContainerProtos.ChunkInfo.newBuilder()
- .setChunkName(name).setOffset(offset).setLen(len).build();
+ .setChunkName(name)
+ .setOffset(offset)
+ .setLen(len)
+ .setChecksumData(Checksum.getNoChecksumDataProto())
+ .build();
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3b03b7/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 26a4ac1..a587ca3 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -132,9 +132,10 @@ public final class DistributedStorageHandler implements StorageHandler {
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
- int configuredChecksumSize = conf.getInt(
+ int configuredChecksumSize = (int) conf.getStorageSize(
OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM,
- OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT);
+ OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT,
+ StorageUnit.BYTES);
int checksumSize;
if(configuredChecksumSize <
OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDDS-886. Unnecessary buffer copy
in HddsDispatcher#dispatch. Contributed by Lokesh Jain.
Posted by sh...@apache.org.
HDDS-886. Unnecessary buffer copy in HddsDispatcher#dispatch. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62f82111
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62f82111
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62f82111
Branch: refs/heads/HDFS-12943
Commit: 62f821115be34f26e994790591c235710f0fc224
Parents: 7ccb640
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Nov 30 22:57:29 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Fri Nov 30 22:57:29 2018 +0800
----------------------------------------------------------------------
.../apache/hadoop/ozone/container/common/impl/HddsDispatcher.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f82111/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 24ba784..352cc86 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -134,9 +134,9 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
@Override
public ContainerCommandResponseProto dispatch(
ContainerCommandRequestProto msg) {
+ Preconditions.checkNotNull(msg);
LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
msg.getTraceID());
- Preconditions.checkNotNull(msg.toString());
AuditAction action = ContainerCommandRequestPBHelper.getAuditAction(
msg.getCmdType());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-15922. Revert patch 004.
Posted by sh...@apache.org.
HADOOP-15922. Revert patch 004.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38ea3814
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38ea3814
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38ea3814
Branch: refs/heads/HDFS-12943
Commit: 38ea3814bd0641d895e5d3b7415c6308e7f8491e
Parents: 62f8211
Author: Eric Yang <ey...@apache.org>
Authored: Fri Nov 30 12:49:46 2018 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Nov 30 12:49:46 2018 -0500
----------------------------------------------------------------------
.../DelegationTokenAuthenticationFilter.java | 8 +--
.../hadoop/crypto/key/kms/server/TestKMS.java | 64 --------------------
2 files changed, 1 insertion(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38ea3814/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index 6b3fc1d..5275526 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -51,7 +51,6 @@ import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
-import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.security.Principal;
import java.util.Enumeration;
@@ -231,12 +230,7 @@ public class DelegationTokenAuthenticationFilter
for (NameValuePair nv : list) {
if (DelegationTokenAuthenticatedURL.DO_AS.
equalsIgnoreCase(nv.getName())) {
- String doAsUser = nv.getValue();
- try {
- doAsUser = URLDecoder.decode(nv.getValue(), UTF8_CHARSET.name());
- } finally {
- return doAsUser;
- }
+ return nv.getValue();
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38ea3814/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 064ae83..af59877 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -2115,70 +2115,6 @@ public class TestKMS {
});
}
- @Test
- public void testGetDelegationTokenByProxyUser() throws Exception {
- Configuration conf = new Configuration();
- conf.set("hadoop.security.authentication", "kerberos");
- UserGroupInformation.setConfiguration(conf);
- final File testDir = getTestDir();
-
- conf = createBaseKMSConf(testDir, conf);
- conf.set("hadoop.kms.authentication.type", "kerberos");
- conf.set("hadoop.kms.authentication.kerberos.keytab",
- keytab.getAbsolutePath());
- conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
- conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
- conf.set("hadoop.kms.proxyuser.client.users", "foo/localhost");
- conf.set("hadoop.kms.proxyuser.client.hosts", "localhost");
- conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kcc.ALL",
- "foo/localhost");
-
- writeConf(testDir, conf);
-
- runServer(null, null, testDir, new KMSCallable<Void>() {
- @Override
- public Void call() throws Exception {
- final Configuration conf = new Configuration();
- final URI uri = createKMSUri(getKMSUrl());
-
- // proxyuser client using kerberos credentials
- UserGroupInformation proxyUgi = UserGroupInformation.
- loginUserFromKeytabAndReturnUGI("client/host", keytab.getAbsolutePath());
- UserGroupInformation foo = UserGroupInformation.createProxyUser(
- "foo/localhost", proxyUgi);
- final Credentials credentials = new Credentials();
- foo.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- final KeyProvider kp = createProvider(uri, conf);
- KeyProviderDelegationTokenExtension keyProviderDelegationTokenExtension
- = KeyProviderDelegationTokenExtension
- .createKeyProviderDelegationTokenExtension(kp);
- keyProviderDelegationTokenExtension.addDelegationTokens("client",
- credentials);
- Assert.assertNotNull(kp.createKey("kcc",
- new KeyProvider.Options(conf)));
- return null;
- }
- });
-
- // current user client using token credentials for proxy user
- UserGroupInformation nonKerberosUgi
- = UserGroupInformation.getCurrentUser();
- nonKerberosUgi.addCredentials(credentials);
- nonKerberosUgi.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- final KeyProvider kp = createProvider(uri, conf);
- Assert.assertNotNull(kp.getMetadata("kcc"));
- return null;
- }
- });
- return null;
- }
- });
- }
-
private Configuration setupConfForKerberos(File confDir) throws Exception {
final Configuration conf = createBaseKMSConf(confDir, null);
conf.set("hadoop.security.authentication", "kerberos");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDFS-14106. Refactor
NamenodeFsck#copyBlock. Contributed by Beluga Behr.
Posted by sh...@apache.org.
HDFS-14106. Refactor NamenodeFsck#copyBlock. Contributed by Beluga Behr.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b09cfad4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b09cfad4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b09cfad4
Branch: refs/heads/HDFS-12943
Commit: b09cfad43268765f0b35af115b82ddb8ac87a3a4
Parents: 6d7b44c
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 30 10:47:59 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 30 10:47:59 2018 -0800
----------------------------------------------------------------------
.../hdfs/server/namenode/NamenodeFsck.java | 37 +++++++++++---------
1 file changed, 21 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b09cfad4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index f54b407..b4c0a93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -1091,28 +1091,33 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
deadNodes.add(chosenNode);
}
}
- byte[] buf = new byte[1024];
- int cnt = 0;
- boolean success = true;
- long bytesRead = 0;
+
+ long bytesRead = 0L;
try {
- while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
- fos.write(buf, 0, cnt);
- bytesRead += cnt;
- }
- if ( bytesRead != block.getNumBytes() ) {
- throw new IOException("Recorded block size is " + block.getNumBytes() +
- ", but datanode returned " +bytesRead+" bytes");
- }
+ bytesRead = copyBock(blockReader, fos);
} catch (Exception e) {
- LOG.error("Error reading block", e);
- success = false;
+ throw new Exception("Could not copy block data for " + lblock.getBlock(),
+ e);
} finally {
blockReader.close();
}
- if (!success) {
- throw new Exception("Could not copy block data for " + lblock.getBlock());
+
+ if (bytesRead != block.getNumBytes()) {
+ throw new IOException("Recorded block size is " + block.getNumBytes()
+ + ", but datanode returned " + bytesRead + " bytes");
+ }
+ }
+
+ private long copyBock(BlockReader blockReader, OutputStream os)
+ throws IOException {
+ final byte[] buf = new byte[8192];
+ int cnt = 0;
+ long bytesRead = 0L;
+ while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
+ os.write(buf, 0, cnt);
+ bytesRead += cnt;
}
+ return bytesRead;
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDFS-13818. Extend OIV to detect
FSImage corruption. Contributed by Adam Antal.
Posted by sh...@apache.org.
HDFS-13818. Extend OIV to detect FSImage corruption. Contributed by Adam Antal.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb10803d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb10803d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb10803d
Branch: refs/heads/HDFS-12943
Commit: fb10803dfa67394650072bdea327296f8ad2a744
Parents: dd5e7c6
Author: Adam Antal <ad...@cloudera.com>
Authored: Mon Dec 3 10:33:51 2018 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Dec 3 10:34:39 2018 -0800
----------------------------------------------------------------------
.../OfflineImageViewerPB.java | 27 +-
.../offlineImageViewer/PBImageCorruption.java | 107 ++++++
.../PBImageCorruptionDetector.java | 344 +++++++++++++++++++
.../PBImageDelimitedTextWriter.java | 38 +-
.../offlineImageViewer/PBImageTextWriter.java | 220 ++++++++++--
.../src/site/markdown/HdfsImageViewer.md | 36 +-
.../TestOfflineImageViewer.java | 219 +++++++++++-
.../TestPBImageCorruption.java | 55 +++
.../test/resources/testMultipleCorruption.csv | 22 ++
.../resources/testMultipleFileCorruption.csv | 21 ++
.../test/resources/testSingleFileCorruption.csv | 18 +
.../resources/testSingleFolderCorruption.csv | 18 +
12 files changed, 1052 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index 782ace8..33b9364 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -79,6 +79,12 @@ public class OfflineImageViewerPB {
+ " to both inodes and inodes-under-construction, separated by a\n"
+ " delimiter. The default delimiter is \\t, though this may be\n"
+ " changed via the -delimiter argument.\n"
+ + " * DetectCorruption: Detect potential corruption of the image by\n"
+ + " selectively loading parts of it and actively searching for\n"
+ + " inconsistencies. Outputs a summary of the found corruptions\n"
+ + " in a delimited format.\n"
+ + " Note that the check is not exhaustive, and only catches\n"
+ + " missing nodes during the namespace reconstruction.\n"
+ "\n"
+ "Required command line arguments:\n"
+ "-i,--inputFile <arg> FSImage or XML file to process.\n"
@@ -91,12 +97,15 @@ public class OfflineImageViewerPB {
+ " will also create an <outputFile>.md5 file.\n"
+ "-p,--processor <arg> Select which type of processor to apply\n"
+ " against image file. (XML|FileDistribution|\n"
- + " ReverseXML|Web|Delimited)\n"
+ + " ReverseXML|Web|Delimited|DetectCorruption)\n"
+ " The default is Web.\n"
- + "-delimiter <arg> Delimiting string to use with Delimited processor. \n"
- + "-t,--temp <arg> Use temporary dir to cache intermediate result to generate\n"
- + " Delimited outputs. If not set, Delimited processor constructs\n"
- + " the namespace in memory before outputting text.\n"
+ + "-delimiter <arg> Delimiting string to use with Delimited or \n"
+ + " DetectCorruption processor. \n"
+ + "-t,--temp <arg> Use temporary dir to cache intermediate\n"
+ + " result to generate DetectCorruption or\n"
+ + " Delimited outputs. If not set, the processor\n"
+ + " constructs the namespace in memory \n"
+ + " before outputting text.\n"
+ "-h,--help Display usage information and exit\n";
/**
@@ -172,7 +181,7 @@ public class OfflineImageViewerPB {
String processor = cmd.getOptionValue("p", "Web");
String outputFile = cmd.getOptionValue("o", "-");
String delimiter = cmd.getOptionValue("delimiter",
- PBImageDelimitedTextWriter.DEFAULT_DELIMITER);
+ PBImageTextWriter.DEFAULT_DELIMITER);
String tempPath = cmd.getOptionValue("t", "");
Configuration conf = new Configuration();
@@ -219,6 +228,12 @@ public class OfflineImageViewerPB {
writer.visit(r);
}
break;
+ case "DETECTCORRUPTION":
+ try (PBImageCorruptionDetector detector =
+ new PBImageCorruptionDetector(out, delimiter, tempPath)) {
+ detector.visit(new RandomAccessFile(inputFile, "r"));
+ }
+ break;
default:
System.err.println("Invalid processor specified : " + processor);
printUsage();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruption.java
new file mode 100644
index 0000000..d510dfc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruption.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.util.EnumSet;
+
+/**
+ * Class representing a corruption in the PBImageCorruptionDetector processor.
+ */
+public class PBImageCorruption {
+ private static final String WITH = "With";
+
+ /**
+ * PBImageCorruptionType is a wrapper for getting a string output for
+ * different types of corruption. Could be added more cases if
+ * other types are revealed. Currently hasMissingChild and
+ * isCorruptNode are the relevant cases.
+ */
+ private enum PBImageCorruptionType {
+ CORRUPT_NODE("CorruptNode"),
+ MISSING_CHILD("MissingChild");
+
+ private final String name;
+
+ PBImageCorruptionType(String s) {
+ name = s;
+ }
+
+ @Override
+ public String toString() {
+ return this.name;
+ }
+ }
+
+ private long id;
+ private EnumSet<PBImageCorruptionType> type;
+ private int numOfCorruptChildren;
+
+ PBImageCorruption(long id, boolean missingChild, boolean corruptNode,
+ int numOfCorruptChildren) {
+ if (!missingChild && !corruptNode) {
+ throw new IllegalArgumentException(
+ "Corruption must have at least one aspect!");
+ }
+ this.id = id;
+ this.type = EnumSet.noneOf(PBImageCorruptionType.class);
+ if (missingChild) {
+ type.add(PBImageCorruptionType.MISSING_CHILD);
+ }
+ if (corruptNode) {
+ type.add(PBImageCorruptionType.CORRUPT_NODE);
+ }
+ this.numOfCorruptChildren = numOfCorruptChildren;
+ }
+
+ void addMissingChildCorruption() {
+ type.add(PBImageCorruptionType.MISSING_CHILD);
+ }
+
+ void addCorruptNodeCorruption() {
+ type.add(PBImageCorruptionType.CORRUPT_NODE);
+ }
+
+ void setNumberOfCorruption(int numOfCorruption) {
+ this.numOfCorruptChildren = numOfCorruption;
+ }
+
+ long getId() {
+ return id;
+ }
+
+ String getType() {
+ StringBuffer s = new StringBuffer();
+ if (type.contains(PBImageCorruptionType.CORRUPT_NODE)) {
+ s.append(PBImageCorruptionType.CORRUPT_NODE);
+ }
+ if (type.contains(PBImageCorruptionType.CORRUPT_NODE) &&
+ type.contains(PBImageCorruptionType.MISSING_CHILD)) {
+ s.append(WITH);
+ }
+
+ if (type.contains(PBImageCorruptionType.MISSING_CHILD)) {
+ s.append(PBImageCorruptionType.MISSING_CHILD);
+ }
+ return s.toString();
+ }
+
+ int getNumOfCorruptChildren() {
+ return numOfCorruptChildren;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java
new file mode 100644
index 0000000..7e5eb31
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * The PBImageCorruptionDetector detects corruptions in the image.
+ * It produces a file with the found issues similar to the Delimited
+ * processor. The default delimiter is tab, as this is an unlikely value
+ * to be included in an inode path. The delimiter value can be changed
+ * via the constructor.
+ *
+ * It looks for the following kinds of corruptions:
+ * - an INode id is mentioned in the INodeDirectorySection, but not present
+ * in the INodeSection (corrupt INode case)
+ * - an INode has children, but at least one of them is corrupted
+ * (missing children case)
+ * If multiple layers of directory structure are damaged then it is possible
+ * that an INode is corrupted and also having corrupted children.
+ *
+ * Note that the OIV DetectCorruption processor check is not exhaustive,
+ * and only catches the corruptions like above. This processor may be up to
+ * extension in the future when new aspects of corruption are found.
+ */
+public class PBImageCorruptionDetector extends PBImageTextWriter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PBImageCorruptionDetector.class);
+
+ /**
+ * Builder object for producing entries (lines) for
+ * PBImageCorruptionDetector. The isSnapshot field is mandatory.
+ */
+ static class OutputEntryBuilder {
+ private static final String MISSING = "Missing";
+
+ private PBImageCorruptionDetector corrDetector;
+ private PBImageCorruption corruption;
+ private boolean isSnapshot;
+ private String parentPath;
+ private long parentId;
+ private String name;
+ private String nodeType;
+
+ OutputEntryBuilder(PBImageCorruptionDetector corrDetector,
+ boolean isSnapshot) {
+ this.corrDetector = corrDetector;
+ this.isSnapshot = isSnapshot;
+ this.parentId = -1;
+ this.parentPath = "";
+ this.name = "";
+ this.nodeType = "";
+ }
+
+ OutputEntryBuilder setCorruption(PBImageCorruption corr) {
+ this.corruption = corr;
+ return this;
+ }
+
+ OutputEntryBuilder setParentPath(String path) {
+ this.parentPath = path;
+ return this;
+ }
+
+ OutputEntryBuilder setParentId(long id) {
+ this.parentId = id;
+ return this;
+ }
+
+ OutputEntryBuilder setName(String n) {
+ this.name = n;
+ return this;
+ }
+
+ OutputEntryBuilder setNodeType(String nType) {
+ this.nodeType = nType;
+ return this;
+ }
+
+ public String build() {
+ StringBuffer buffer = new StringBuffer();
+ buffer.append(corruption.getType());
+ corrDetector.append(buffer, corruption.getId());
+ corrDetector.append(buffer, String.valueOf(isSnapshot));
+ corrDetector.append(buffer, parentPath);
+ if (parentId == -1) {
+ corrDetector.append(buffer, MISSING);
+ } else {
+ corrDetector.append(buffer, parentId);
+ }
+ corrDetector.append(buffer, name);
+ corrDetector.append(buffer, nodeType);
+ corrDetector.append(buffer, corruption.getNumOfCorruptChildren());
+ return buffer.toString();
+ }
+ }
+
+ private static class CorruptionChecker {
+ private static final String NODE_TYPE = "Node";
+ private static final String REF_TYPE = "Ref";
+ private static final String UNKNOWN_TYPE = "Unknown";
+
+ /** Contains all existing INode IDs. */
+ private Set<Long> nodeIds;
+ /** Contains all existing INodeReference IDs. */
+ private Set<Long> nodeRefIds;
+
+ CorruptionChecker() {
+ nodeIds = new HashSet<>();
+ }
+
+ /**
+ * Collect a INode Id.
+ */
+ void saveNodeId(long id) {
+ Preconditions.checkState(nodeIds != null && !nodeIds.contains(id));
+ nodeIds.add(id);
+ }
+
+ /**
+ * Returns whether the given INode id was saved previously.
+ */
+ boolean isNodeIdExist(long id) {
+ return nodeIds.contains(id);
+ }
+
+ /**
+ * Returns whether the given INodeReference id was saved previously.
+ */
+ boolean isNodeRefIdExist(long id) {
+ return nodeRefIds.contains(id);
+ }
+
+ /**
+ * Saves the INodeReference ids.
+ */
+ void saveNodeRefIds(List<Long> nodeRefIdList) {
+ nodeRefIds = new HashSet<>(nodeRefIdList);
+ }
+
+ String getTypeOfId(long id) {
+ if (isNodeIdExist(id)) {
+ return NODE_TYPE;
+ } else if (isNodeRefIdExist(id)) {
+ return REF_TYPE;
+ } else {
+ return UNKNOWN_TYPE;
+ }
+ }
+ }
+
+ /** Delimiter string used while producing output. */
+ private final CorruptionChecker corrChecker;
+ /** Id to corruption mapping. */
+ private final Map<Long, PBImageCorruption> corruptionsMap;
+
+ PBImageCorruptionDetector(PrintStream out, String delimiter,
+ String tempPath) throws IOException {
+ super(out, delimiter, tempPath);
+ corrChecker = new CorruptionChecker();
+ corruptionsMap = new TreeMap<Long, PBImageCorruption>();
+ }
+
+ @Override
+ public String getHeader() {
+ StringBuffer buffer = new StringBuffer();
+ buffer.append("CorruptionType");
+ append(buffer, "Id");
+ append(buffer, "IsSnapshot");
+ append(buffer, "ParentPath");
+ append(buffer, "ParentId");
+ append(buffer, "Name");
+ append(buffer, "NodeType");
+ append(buffer, "CorruptChildren");
+ return buffer.toString();
+ }
+
+ @Override
+ public String getEntry(String parentPath,
+ FsImageProto.INodeSection.INode inode) {
+ long id = inode.getId();
+ if (corruptionsMap.containsKey(id)) {
+ OutputEntryBuilder entryBuilder =
+ new OutputEntryBuilder(this, false);
+ long parentId = -1;
+ try {
+ parentId = getParentId(id);
+ } catch (IOException ignore) {
+ }
+ entryBuilder.setCorruption(corruptionsMap.get(id))
+ .setParentPath(parentPath)
+ .setName(inode.getName().toStringUtf8())
+ .setNodeType(corrChecker.getTypeOfId(id));
+ if (parentId != -1) {
+ entryBuilder.setParentId(parentId);
+ }
+ corruptionsMap.remove(id);
+ return entryBuilder.build();
+ } else {
+ return "";
+ }
+ }
+
+ @Override
+ protected void checkNode(FsImageProto.INodeSection.INode p,
+ AtomicInteger numDirs) throws IOException {
+ super.checkNode(p, numDirs);
+ corrChecker.saveNodeId(p.getId());
+ }
+
+ private void addCorruptedNode(long childId) {
+ if (!corruptionsMap.containsKey(childId)) {
+ PBImageCorruption c = new PBImageCorruption(childId, false, true, 0);
+ corruptionsMap.put(childId, c);
+ } else {
+ PBImageCorruption c = corruptionsMap.get(childId);
+ c.addCorruptNodeCorruption();
+ corruptionsMap.put(childId, c);
+ }
+ }
+
+ private void addCorruptedParent(long id, int numOfCorruption) {
+ if (!corruptionsMap.containsKey(id)) {
+ PBImageCorruption c = new PBImageCorruption(id, true, false,
+ numOfCorruption);
+ corruptionsMap.put(id, c);
+ } else {
+ PBImageCorruption c = corruptionsMap.get(id);
+ c.addMissingChildCorruption();
+ c.setNumberOfCorruption(numOfCorruption);
+ corruptionsMap.put(id, c);
+ }
+ }
+
+ /**
+ * Scan the INodeDirectory section to construct the namespace.
+ */
+ @Override
+ protected void buildNamespace(InputStream in, List<Long> refIdList)
+ throws IOException {
+ corrChecker.saveNodeRefIds(refIdList);
+ LOG.debug("Saved INodeReference ids of size {}.", refIdList.size());
+ int count = 0;
+ while (true) {
+ FsImageProto.INodeDirectorySection.DirEntry e =
+ FsImageProto.INodeDirectorySection.DirEntry.parseDelimitedFrom(in);
+ if (e == null) {
+ break;
+ }
+ count++;
+ if (LOG.isDebugEnabled() && count % 10000 == 0) {
+ LOG.debug("Scanned {} directories.", count);
+ }
+ long parentId = e.getParent();
+ if (!corrChecker.isNodeIdExist(parentId)) {
+ LOG.debug("Corruption detected! Parent node is not contained " +
+ "in the list of known ids!");
+ addCorruptedNode(parentId);
+ }
+ int numOfCorruption = 0;
+ for (int i = 0; i < e.getChildrenCount(); i++) {
+ long childId = e.getChildren(i);
+ putDirChildToMetadataMap(parentId, childId);
+ if (!corrChecker.isNodeIdExist(childId)) {
+ addCorruptedNode(childId);
+ numOfCorruption++;
+ }
+ }
+ if (numOfCorruption > 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{} corruption detected! Child nodes are missing.",
+ numOfCorruption);
+ }
+ addCorruptedParent(parentId, numOfCorruption);
+ }
+ for (int i = e.getChildrenCount();
+ i < e.getChildrenCount() + e.getRefChildrenCount(); i++) {
+ int refId = e.getRefChildren(i - e.getChildrenCount());
+ // In this case the refNode is referred directly (by its position),
+ // so we couldn't make sure of the correctness
+ putDirChildToMetadataMap(parentId, refIdList.get(refId));
+ }
+ }
+ LOG.info("Scanned {} INode directories to build namespace.", count);
+ }
+
+ @Override
+ public void afterOutput() throws IOException {
+ if (!corruptionsMap.isEmpty()) {
+ // Also write out corruptions when the path could be not be decided
+ LOG.info("Outputting {} more corrupted nodes.", corruptionsMap.size());
+ for (PBImageCorruption c : corruptionsMap.values()) {
+ long id = c.getId();
+ String name = "";
+ long parentId = -1;
+ try {
+ name = getNodeName(id);
+ } catch (IgnoreSnapshotException ignored) {
+ }
+ try {
+ parentId = getParentId(id);
+ } catch (IgnoreSnapshotException ignored) {
+ }
+ OutputEntryBuilder entryBuilder =
+ new OutputEntryBuilder(this, true);
+ entryBuilder.setCorruption(corruptionsMap.get(id))
+ .setName(name)
+ .setNodeType(corrChecker.getTypeOfId(id));
+ if (parentId != -1) {
+ entryBuilder.setParentId(parentId);
+ }
+ printIfNotEmpty(entryBuilder.build());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
index 20a8350..7b48451 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.text.StringEscapeUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
@@ -46,48 +44,19 @@ import java.util.Date;
* constructor.
*/
public class PBImageDelimitedTextWriter extends PBImageTextWriter {
- static final String DEFAULT_DELIMITER = "\t";
private static final String DATE_FORMAT="yyyy-MM-dd HH:mm";
private final SimpleDateFormat dateFormatter =
new SimpleDateFormat(DATE_FORMAT);
- private final String delimiter;
-
PBImageDelimitedTextWriter(PrintStream out, String delimiter, String tempPath)
throws IOException {
- super(out, tempPath);
- this.delimiter = delimiter;
+ super(out, delimiter, tempPath);
}
private String formatDate(long date) {
return dateFormatter.format(new Date(date));
}
- private void append(StringBuffer buffer, int field) {
- buffer.append(delimiter);
- buffer.append(field);
- }
-
- private void append(StringBuffer buffer, long field) {
- buffer.append(delimiter);
- buffer.append(field);
- }
-
- static final String CRLF = StringUtils.CR + StringUtils.LF;
-
- private void append(StringBuffer buffer, String field) {
- buffer.append(delimiter);
-
- String escapedField = StringEscapeUtils.escapeCsv(field);
- if (escapedField.contains(CRLF)) {
- escapedField = escapedField.replace(CRLF, "%x0D%x0A");
- } else if (escapedField.contains(StringUtils.LF)) {
- escapedField = escapedField.replace(StringUtils.LF, "%x0A");
- }
-
- buffer.append(escapedField);
- }
-
@Override
public String getEntry(String parent, INode inode) {
StringBuffer buffer = new StringBuffer();
@@ -168,4 +137,9 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
append(buffer, "GroupName");
return buffer.toString();
}
+
+ @Override
+ public void afterOutput() {
+ // do nothing
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
index ee78395..6f36be4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.PermissionStatus;
@@ -56,6 +58,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -88,6 +91,9 @@ abstract class PBImageTextWriter implements Closeable {
private static final Logger LOG =
LoggerFactory.getLogger(PBImageTextWriter.class);
+ static final String DEFAULT_DELIMITER = "\t";
+ static final String CRLF = StringUtils.CR + StringUtils.LF;
+
/**
* This metadata map is used to construct the namespace before generating
* text outputs.
@@ -114,6 +120,15 @@ abstract class PBImageTextWriter implements Closeable {
/** Synchronize metadata to persistent storage, if possible */
public void sync() throws IOException;
+
+ /** Returns the name of inode. */
+ String getName(long id) throws IOException;
+
+ /**
+ * Returns the id of the parent's inode, if mentioned in
+ * INodeDirectorySection, throws IgnoreSnapshotException otherwise.
+ */
+ long getParentId(long id) throws IOException;
}
/**
@@ -142,18 +157,32 @@ abstract class PBImageTextWriter implements Closeable {
/**
* Returns the full path of this directory.
*/
- private String getPath() {
+ String getPath() throws IgnoreSnapshotException {
if (this.parent == null) {
- return "/";
+ if (this.inode == INodeId.ROOT_INODE_ID) {
+ return "/";
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not root inode with id {} having no parent.", inode);
+ }
+ throw PBImageTextWriter.createIgnoredSnapshotException(inode);
+ }
}
if (this.path == null) {
this.path = new Path(parent.getPath(), name.isEmpty() ? "/" : name).
toString();
- this.name = null;
}
return this.path;
}
+ String getName() throws IgnoreSnapshotException {
+ return name;
+ }
+
+ long getId() {
+ return inode;
+ }
+
@Override
public boolean equals(Object o) {
return o instanceof Dir && inode == ((Dir) o).inode;
@@ -165,6 +194,29 @@ abstract class PBImageTextWriter implements Closeable {
}
}
+ /**
+ * If the Dir entry does not exist (i.e. the inode was not contained in
+ * INodeSection) we still create a Dir entry which throws exceptions
+ * for calls other than getId().
+ * We can make sure this way, the getId and getParentId calls will
+ * always succeed if we have the information.
+ */
+ private static class CorruptedDir extends Dir {
+ CorruptedDir(long inode) {
+ super(inode, null);
+ }
+
+ @Override
+ String getPath() throws IgnoreSnapshotException {
+ throw PBImageTextWriter.createIgnoredSnapshotException(getId());
+ }
+
+ @Override
+ String getName() throws IgnoreSnapshotException {
+ throw PBImageTextWriter.createIgnoredSnapshotException(getId());
+ }
+ }
+
/** INode Id to Dir object mapping */
private Map<Long, Dir> dirMap = new HashMap<>();
@@ -178,13 +230,20 @@ abstract class PBImageTextWriter implements Closeable {
public void close() throws IOException {
}
+ private Dir getOrCreateCorrupted(long id) {
+ Dir dir = dirMap.get(id);
+ if (dir == null) {
+ dir = new CorruptedDir(id);
+ dirMap.put(id, dir);
+ }
+ return dir;
+ }
+
@Override
public void putDirChild(long parentId, long childId) {
- Dir parent = dirMap.get(parentId);
- Dir child = dirMap.get(childId);
- if (child != null) {
- child.setParent(parent);
- }
+ Dir parent = getOrCreateCorrupted(parentId);
+ Dir child = getOrCreateCorrupted(childId);
+ child.setParent(parent);
Preconditions.checkState(!dirChildMap.containsKey(childId));
dirChildMap.put(childId, parent);
}
@@ -199,13 +258,13 @@ abstract class PBImageTextWriter implements Closeable {
@Override
public String getParentPath(long inode) throws IOException {
if (inode == INodeId.ROOT_INODE_ID) {
- return "";
+ return "/";
}
Dir parent = dirChildMap.get(inode);
if (parent == null) {
// The inode is an INodeReference, which is generated from snapshot.
// For delimited oiv tool, no need to print out metadata in snapshots.
- PBImageTextWriter.ignoreSnapshotName(inode);
+ throw PBImageTextWriter.createIgnoredSnapshotException(inode);
}
return parent.getPath();
}
@@ -213,6 +272,24 @@ abstract class PBImageTextWriter implements Closeable {
@Override
public void sync() {
}
+
+ @Override
+ public String getName(long id) throws IgnoreSnapshotException {
+ Dir dir = dirMap.get(id);
+ if (dir != null) {
+ return dir.getName();
+ }
+ throw PBImageTextWriter.createIgnoredSnapshotException(id);
+ }
+
+ @Override
+ public long getParentId(long id) throws IgnoreSnapshotException {
+ Dir parentDir = dirChildMap.get(id);
+ if (parentDir != null) {
+ return parentDir.getId();
+ }
+ throw PBImageTextWriter.createIgnoredSnapshotException(id);
+ }
}
/**
@@ -353,33 +430,37 @@ abstract class PBImageTextWriter implements Closeable {
dirMap.put(toBytes(dir.getId()), toBytes(dir.getName().toStringUtf8()));
}
- @Override
- public String getParentPath(long inode) throws IOException {
- if (inode == INodeId.ROOT_INODE_ID) {
- return "/";
- }
+ private long getFromDirChildMap(long inode) throws IOException {
byte[] bytes = dirChildMap.get(toBytes(inode));
if (bytes == null) {
// The inode is an INodeReference, which is generated from snapshot.
// For delimited oiv tool, no need to print out metadata in snapshots.
- PBImageTextWriter.ignoreSnapshotName(inode);
+ throw PBImageTextWriter.createIgnoredSnapshotException(inode);
}
if (bytes.length != 8) {
throw new IOException(
"bytes array length error. Actual length is " + bytes.length);
}
- long parent = toLong(bytes);
+ return toLong(bytes);
+ }
+
+ @Override
+ public String getParentPath(long inode) throws IOException {
+ if (inode == INodeId.ROOT_INODE_ID) {
+ return "/";
+ }
+ long parent = getFromDirChildMap(inode);
if (!dirPathCache.containsKey(parent)) {
- bytes = dirMap.get(toBytes(parent));
+ byte[] bytes = dirMap.get(toBytes(parent));
if (parent != INodeId.ROOT_INODE_ID && bytes == null) {
// The parent is an INodeReference, which is generated from snapshot.
// For delimited oiv tool, no need to print out metadata in snapshots.
- PBImageTextWriter.ignoreSnapshotName(parent);
+ throw PBImageTextWriter.createIgnoredSnapshotException(inode);
}
String parentName = toString(bytes);
String parentPath =
new Path(getParentPath(parent),
- parentName.isEmpty()? "/" : parentName).toString();
+ parentName.isEmpty() ? "/" : parentName).toString();
dirPathCache.put(parent, parentPath);
}
return dirPathCache.get(parent);
@@ -390,11 +471,26 @@ abstract class PBImageTextWriter implements Closeable {
dirChildMap.sync();
dirMap.sync();
}
+
+ @Override
+ public String getName(long id) throws IOException {
+ byte[] bytes = dirMap.get(toBytes(id));
+ if (bytes != null) {
+ return toString(bytes);
+ }
+ throw PBImageTextWriter.createIgnoredSnapshotException(id);
+ }
+
+ @Override
+ public long getParentId(long id) throws IOException {
+ return getFromDirChildMap(id);
+ }
}
private SerialNumberManager.StringTable stringTable;
private PrintStream out;
private MetadataMap metadataMap = null;
+ private String delimiter;
/**
* Construct a PB FsImage writer to generate text file.
@@ -402,8 +498,10 @@ abstract class PBImageTextWriter implements Closeable {
* @param tempPath the path to store metadata. If it is empty, store metadata
* in memory instead.
*/
- PBImageTextWriter(PrintStream out, String tempPath) throws IOException {
+ PBImageTextWriter(PrintStream out, String delimiter, String tempPath)
+ throws IOException {
this.out = out;
+ this.delimiter = delimiter;
if (tempPath.isEmpty()) {
metadataMap = new InMemoryMetadataDB();
} else {
@@ -417,6 +515,29 @@ abstract class PBImageTextWriter implements Closeable {
IOUtils.cleanup(null, metadataMap);
}
+ void append(StringBuffer buffer, int field) {
+ buffer.append(delimiter);
+ buffer.append(field);
+ }
+
+ void append(StringBuffer buffer, long field) {
+ buffer.append(delimiter);
+ buffer.append(field);
+ }
+
+ void append(StringBuffer buffer, String field) {
+ buffer.append(delimiter);
+
+ String escapedField = StringEscapeUtils.escapeCsv(field);
+ if (escapedField.contains(CRLF)) {
+ escapedField = escapedField.replace(CRLF, "%x0D%x0A");
+ } else if (escapedField.contains(StringUtils.LF)) {
+ escapedField = escapedField.replace(StringUtils.LF, "%x0A");
+ }
+
+ buffer.append(escapedField);
+ }
+
/**
* Get text output for the given inode.
* @param parent the path of parent directory
@@ -429,6 +550,13 @@ abstract class PBImageTextWriter implements Closeable {
*/
abstract protected String getHeader();
+ /**
+ * Method called at the end of output() phase after all the inodes
+ * with known parentPath has been printed out. Can be used to print
+ * additional data depending on the written inodes.
+ */
+ abstract protected void afterOutput() throws IOException;
+
public void visit(RandomAccessFile file) throws IOException {
Configuration conf = new Configuration();
if (!FSImageUtil.checkFileFormat(file)) {
@@ -489,6 +617,19 @@ abstract class PBImageTextWriter implements Closeable {
}
}
+ void putDirChildToMetadataMap(long parentId, long childId)
+ throws IOException {
+ metadataMap.putDirChild(parentId, childId);
+ }
+
+ String getNodeName(long id) throws IOException {
+ return metadataMap.getName(id);
+ }
+
+ long getParentId(long id) throws IOException {
+ return metadataMap.getParentId(id);
+ }
+
private void output(Configuration conf, FileSummary summary,
FileInputStream fin, ArrayList<FileSummary.Section> sections)
throws IOException {
@@ -504,6 +645,7 @@ abstract class PBImageTextWriter implements Closeable {
outputINodes(is);
}
}
+ afterOutput();
long timeTaken = Time.monotonicNow() - startTime;
LOG.debug("Time to output inodes: {}ms", timeTaken);
}
@@ -554,21 +696,30 @@ abstract class PBImageTextWriter implements Closeable {
}
/**
+ * Checks the inode (saves if directory), and counts them. Can be overridden
+ * if additional steps are taken when iterating through INodeSection.
+ */
+ protected void checkNode(INode p, AtomicInteger numDirs) throws IOException {
+ if (p.hasDirectory()) {
+ metadataMap.putDir(p);
+ numDirs.incrementAndGet();
+ }
+ }
+
+ /**
* Load the filenames of the directories from the INode section.
*/
- private void loadDirectoriesInINodeSection(InputStream in) throws IOException {
+ private void loadDirectoriesInINodeSection(InputStream in)
+ throws IOException {
INodeSection s = INodeSection.parseDelimitedFrom(in);
LOG.info("Loading directories in INode section.");
- int numDirs = 0;
+ AtomicInteger numDirs = new AtomicInteger(0);
for (int i = 0; i < s.getNumInodes(); ++i) {
INode p = INode.parseDelimitedFrom(in);
if (LOG.isDebugEnabled() && i % 10000 == 0) {
LOG.debug("Scanned {} inodes.", i);
}
- if (p.hasDirectory()) {
- metadataMap.putDir(p);
- numDirs++;
- }
+ checkNode(p, numDirs);
}
LOG.info("Found {} directories in INode section.", numDirs);
}
@@ -576,7 +727,7 @@ abstract class PBImageTextWriter implements Closeable {
/**
* Scan the INodeDirectory section to construct the namespace.
*/
- private void buildNamespace(InputStream in, List<Long> refIdList)
+ protected void buildNamespace(InputStream in, List<Long> refIdList)
throws IOException {
int count = 0;
while (true) {
@@ -603,6 +754,12 @@ abstract class PBImageTextWriter implements Closeable {
LOG.info("Scanned {} INode directories to build namespace.", count);
}
+ void printIfNotEmpty(String line) {
+ if (!line.isEmpty()) {
+ out.println(line);
+ }
+ }
+
private void outputINodes(InputStream in) throws IOException {
INodeSection s = INodeSection.parseDelimitedFrom(in);
LOG.info("Found {} INodes in the INode section", s.getNumInodes());
@@ -612,7 +769,7 @@ abstract class PBImageTextWriter implements Closeable {
INode p = INode.parseDelimitedFrom(in);
try {
String parentPath = metadataMap.getParentPath(p.getId());
- out.println(getEntry(parentPath, p));
+ printIfNotEmpty(getEntry(parentPath, p));
} catch (IOException ioe) {
ignored++;
if (!(ioe instanceof IgnoreSnapshotException)) {
@@ -636,11 +793,12 @@ abstract class PBImageTextWriter implements Closeable {
LOG.info("Outputted {} INodes.", s.getNumInodes());
}
- static void ignoreSnapshotName(long inode) throws IOException {
+ private static IgnoreSnapshotException createIgnoredSnapshotException(
+ long inode) {
// Ignore snapshots - we want the output similar to -ls -R.
if (LOG.isDebugEnabled()) {
LOG.debug("No snapshot name found for inode {}", inode);
}
- throw new IgnoreSnapshotException();
+ return new IgnoreSnapshotException();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index 6b0c27c..7e3cd73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -55,7 +55,13 @@ The Offline Image Viewer provides several output processors:
delimiter. The default delimiter is \t, though this may be changed via
the -delimiter argument.
-5. ReverseXML (experimental): This is the opposite of the XML processor;
+5. DetectCorruption (experimental): Detect potential corruption of the image
+ by selectively loading parts of it and actively searching for
+ inconsistencies. Outputs a summary of the found corruptions
+ in a delimited format. Note that the check is not exhaustive,
+ and only catches missing nodes during the namespace reconstruction.
+
+6. ReverseXML (experimental): This is the opposite of the XML processor;
it reconstructs an fsimage from an XML file. This processor makes it easy to
create fsimages for testing, and manually edit fsimages when there is
corruption.
@@ -197,6 +203,30 @@ If not set, Delimited processor will construct the namespace in memory before ou
/dir0/file1 1 2017-02-13 10:39 2017-02-13 10:39 134217728 1 1 0 0 -rw-r--r-- root supergroup
/dir0/file2 1 2017-02-13 10:39 2017-02-13 10:39 134217728 1 1 0 0 -rw-r--r-- root supergroup
+### DetectCorruption Processor
+
+DetectCorruption processor generates a text representation of the errors of the fsimage, if there's any. It displays the following cases:
+
+1. an inode is mentioned in the fsimage but no associated metadata is found (CorruptNode)
+
+2. an inode has at least one corrupt children (MissingChildren)
+
+The delimiter string can be provided with the -delimiter option, and the processor can cache intermediate result using the -t option.
+
+ bash$ bin/hdfs oiv -p DetectCorruption -delimiter delimiterString -t temporaryDir -i fsimage -o output
+
+The output result of this processor is empty if no corruption is found, otherwise the found entries in the following format:
+
+ CorruptionType Id IsSnapshot ParentPath ParentId Name NodeType CorruptChildren
+ MissingChild 16385 false / Missing Node 1
+ MissingChild 16386 false / 16385 dir0 Node 2
+ CorruptNode 16388 true 16386 Unknown 0
+ CorruptNode 16389 true 16386 Unknown 0
+ CorruptNodeWithMissingChild 16391 true 16385 Unknown 1
+ CorruptNode 16394 true 16391 Unknown 0
+
+The column CorruptionType can be MissingChild, CorruptNode or the combination of these two. IsSnapshot shows whether the node is kept in a snapshot or not. To the NodeType column either Node, Ref or Unknown can be written depending whether the node is an inode, a reference, or is corrupted and thus unknown. CorruptChildren contains the number of the corrupt children the inode may have.
+
Options
-------
@@ -204,12 +234,12 @@ Options
|:---- |:---- |
| `-i`\|`--inputFile` *input file* | Specify the input fsimage file (or XML file, if ReverseXML processor is used) to process. Required. |
| `-o`\|`--outputFile` *output file* | Specify the output filename, if the specified output processor generates one. If the specified file already exists, it is silently overwritten. (output to stdout by default) If the input file is an XML file, it also creates an <outputFile>.md5. |
-| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Currently valid options are `Web` (default), `XML`, `Delimited`, `FileDistribution` and `ReverseXML`. |
+| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Currently valid options are `Web` (default), `XML`, `Delimited`, `DetectCorruption`, `FileDistribution` and `ReverseXML`. |
| `-addr` *address* | Specify the address(host:port) to listen. (localhost:5978 by default). This option is used with Web processor. |
| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
-| `-delimiter` *arg* | Delimiting string to use with Delimited processor. |
+| `-delimiter` *arg* | Delimiting string to use with Delimited or DetectCorruption processor. |
| `-t`\|`--temp` *temporary dir* | Use temporary dir to cache intermediate result to generate Delimited outputs. If not set, Delimited processor constructs the namespace in memory before outputting text. |
| `-h`\|`--help` | Display the tool usage and help information and exit. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 8c8d404..a01de74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -43,6 +43,7 @@ import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.E
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.BufferedReader;
@@ -51,6 +52,7 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
@@ -60,11 +62,13 @@ import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -76,6 +80,10 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.output.NullOutputStream;
@@ -134,6 +142,10 @@ public class TestOfflineImageViewer {
private static File originalFsimage = null;
private static int filesECCount = 0;
private static String addedErasureCodingPolicyName = null;
+ private static final long FILE_NODE_ID_1 = 16388;
+ private static final long FILE_NODE_ID_2 = 16389;
+ private static final long FILE_NODE_ID_3 = 16394;
+ private static final long DIR_NODE_ID = 16391;
// namespace as written to dfs, to be compared with viewer's output
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
@@ -636,6 +648,39 @@ public class TestOfflineImageViewer {
}
@Test
+ public void testOutputEntryBuilder() throws IOException {
+ PBImageCorruptionDetector corrDetector =
+ new PBImageCorruptionDetector(null, ",", "");
+ PBImageCorruption c1 = new PBImageCorruption(342, true, false, 3);
+ PBImageCorruptionDetector.OutputEntryBuilder entryBuilder1 =
+ new PBImageCorruptionDetector.OutputEntryBuilder(corrDetector, false);
+ entryBuilder1.setParentId(1)
+ .setCorruption(c1)
+ .setParentPath("/dir1/dir2/");
+ assertEquals(entryBuilder1.build(),
+ "MissingChild,342,false,/dir1/dir2/,1,,,3");
+
+ corrDetector = new PBImageCorruptionDetector(null, "\t", "");
+ PBImageCorruption c2 = new PBImageCorruption(781, false, true, 0);
+ PBImageCorruptionDetector.OutputEntryBuilder entryBuilder2 =
+ new PBImageCorruptionDetector.OutputEntryBuilder(corrDetector, true);
+ entryBuilder2.setParentPath("/dir3/")
+ .setCorruption(c2)
+ .setName("folder")
+ .setNodeType("Node");
+ assertEquals(entryBuilder2.build(),
+ "CorruptNode\t781\ttrue\t/dir3/\tMissing\tfolder\tNode\t0");
+ }
+
+ @Test
+ public void testPBCorruptionDetector() throws IOException,
+ InterruptedException {
+ testPBCorruptionDetector(""); // Test in memory db.
+ testPBCorruptionDetector(
+ new FileSystemTestHelper().getTestRootDir() + "/corruption.db");
+ }
+
+ @Test
public void testInvalidProcessorOption() throws Exception {
int status =
OfflineImageViewerPB.run(new String[] { "-i",
@@ -720,7 +765,7 @@ public class TestOfflineImageViewer {
}
// writtenFiles does not contain root directory and "invalid XML char" dir.
- for (Iterator<String> it = fileNames.iterator(); it.hasNext(); ) {
+ for (Iterator<String> it = fileNames.iterator(); it.hasNext();) {
String filename = it.next();
if (filename.startsWith("/dirContainingInvalidXMLChar")) {
it.remove();
@@ -731,6 +776,178 @@ public class TestOfflineImageViewer {
assertEquals(writtenFiles.keySet(), fileNames);
}
+ private void testPBCorruptionDetector(String db)
+ throws IOException, InterruptedException {
+ final String delimiter = "\t";
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+ try (PrintStream o = new PrintStream(output)) {
+ PBImageCorruptionDetector v =
+ new PBImageCorruptionDetector(o, delimiter, db);
+ v.visit(new RandomAccessFile(originalFsimage, "r"));
+ }
+
+ try (
+ ByteArrayInputStream input =
+ new ByteArrayInputStream(output.toByteArray());
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(input))) {
+ String line = reader.readLine();
+ System.out.println(line);
+ String[] fields = line.split(delimiter);
+ assertEquals(8, fields.length);
+ PBImageCorruptionDetector v =
+ new PBImageCorruptionDetector(null, delimiter, "");
+ assertEquals(line, v.getHeader());
+ line = reader.readLine();
+ assertNull(line);
+ }
+ }
+
+ private void properINodeDelete(List<Long> idsToDelete, Document doc)
+ throws IOException {
+ NodeList inodes = doc.getElementsByTagName("id");
+ if (inodes.getLength() < 1) {
+ throw new IOException("No id tags found in the image xml.");
+ }
+ for (long idToDelete : idsToDelete) {
+ boolean found = false;
+ for (int i = 0; i < inodes.getLength(); i++) {
+ Node id = inodes.item(i);
+ if (id.getTextContent().equals(String.valueOf(idToDelete))) {
+ found = true;
+ Node inode = id.getParentNode();
+ Node inodeSection = inode.getParentNode();
+ inodeSection.removeChild(inode);
+ break;
+ }
+ }
+ if (!found) {
+ throw new IOException("Couldn't find the id in the image.");
+ }
+ }
+ NodeList numInodesNodes = doc.getElementsByTagName("numInodes");
+ if (numInodesNodes.getLength() != 1) {
+ throw new IOException("More than one numInodes tag found.");
+ }
+ Node numInodesNode = numInodesNodes.item(0);
+ int numberOfINodes = Integer.parseInt(numInodesNode.getTextContent());
+ numberOfINodes -= idsToDelete.size();
+ numInodesNode.setTextContent(String.valueOf(numberOfINodes));
+ }
+
+ private void deleteINodeFromXML(File inputFile, File outputFile,
+ List<Long> corruptibleIds) throws Exception {
+ DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
+ Document doc = docBuilder.parse(inputFile);
+
+ properINodeDelete(corruptibleIds, doc);
+
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(outputFile);
+ transformer.transform(source, result);
+ }
+
+ private void generateMissingNodeCorruption(File goodImageXml,
+ File corruptedImageXml, File corruptedImage, List<Long> corruptibleIds)
+ throws Exception {
+ if (OfflineImageViewerPB.run(new String[] {"-p", "XML",
+ "-i", originalFsimage.getAbsolutePath(),
+ "-o", goodImageXml.getAbsolutePath() }) != 0) {
+ throw new IOException("Couldn't create XML!");
+ }
+ deleteINodeFromXML(goodImageXml, corruptedImageXml, corruptibleIds);
+ if (OfflineImageViewerPB.run(new String[] {"-p", "ReverseXML",
+ "-i", corruptedImageXml.getAbsolutePath(),
+ "-o", corruptedImage.getAbsolutePath() }) != 0) {
+ throw new IOException("Couldn't create from XML!");
+ }
+ }
+
+ private String testCorruptionDetectorRun(int runNumber,
+ List<Long> corruptions, String db) throws Exception {
+ File goodImageXml = new File(tempDir, "goodImage" + runNumber +".xml");
+ File corruptedImageXml = new File(tempDir,
+ "corruptedImage" + runNumber + ".xml");
+ File corruptedImage = new File(originalFsimage.getParent(),
+ "fsimage_corrupted" + runNumber);
+ generateMissingNodeCorruption(goodImageXml, corruptedImageXml,
+ corruptedImage, corruptions);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ try (PrintStream o = new PrintStream(output)) {
+ PBImageCorruptionDetector v =
+ new PBImageCorruptionDetector(o, ",", db);
+ v.visit(new RandomAccessFile(corruptedImage, "r"));
+ }
+ return output.toString();
+ }
+
+ private String readExpectedFile(String fileName) throws IOException {
+ File file = new File(System.getProperty(
+ "test.cache.data", "build/test/cache"), fileName);
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ String line;
+ StringBuilder s = new StringBuilder();
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+ if (line.length() <= 0 || line.startsWith("#")) {
+ continue;
+ }
+ s.append(line);
+ s.append("\n");
+ }
+ return s.toString();
+ }
+
+ @Test
+ public void testCorruptionDetectionSingleFileCorruption() throws Exception {
+ List<Long> corruptions = Collections.singletonList(FILE_NODE_ID_1);
+ String result = testCorruptionDetectorRun(1, corruptions, "");
+ String expected = readExpectedFile("testSingleFileCorruption.csv");
+ assertEquals(expected, result);
+ result = testCorruptionDetectorRun(2, corruptions,
+ new FileSystemTestHelper().getTestRootDir() + "/corruption2.db");
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void testCorruptionDetectionMultipleFileCorruption() throws Exception {
+ List<Long> corruptions = Arrays.asList(FILE_NODE_ID_1, FILE_NODE_ID_2,
+ FILE_NODE_ID_3);
+ String result = testCorruptionDetectorRun(3, corruptions, "");
+ String expected = readExpectedFile("testMultipleFileCorruption.csv");
+ assertEquals(expected, result);
+ result = testCorruptionDetectorRun(4, corruptions,
+ new FileSystemTestHelper().getTestRootDir() + "/corruption4.db");
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void testCorruptionDetectionSingleFolderCorruption() throws Exception {
+ List<Long> corruptions = Collections.singletonList(DIR_NODE_ID);
+ String result = testCorruptionDetectorRun(5, corruptions, "");
+ String expected = readExpectedFile("testSingleFolderCorruption.csv");
+ assertEquals(expected, result);
+ result = testCorruptionDetectorRun(6, corruptions,
+ new FileSystemTestHelper().getTestRootDir() + "/corruption6.db");
+ assertEquals(expected, result);
+ }
+
+ @Test
+ public void testCorruptionDetectionMultipleCorruption() throws Exception {
+ List<Long> corruptions = Arrays.asList(FILE_NODE_ID_1, FILE_NODE_ID_2,
+ FILE_NODE_ID_3, DIR_NODE_ID);
+ String result = testCorruptionDetectorRun(7, corruptions, "");
+ String expected = readExpectedFile("testMultipleCorruption.csv");
+ assertEquals(expected, result);
+ result = testCorruptionDetectorRun(8, corruptions,
+ new FileSystemTestHelper().getTestRootDir() + "/corruption8.db");
+ assertEquals(expected, result);
+ }
+
private static void compareFile(FileStatus expected, FileStatus status) {
assertEquals(expected.getAccessTime(), status.getAccessTime());
assertEquals(expected.getBlockSize(), status.getBlockSize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestPBImageCorruption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestPBImageCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestPBImageCorruption.java
new file mode 100644
index 0000000..092d3a1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestPBImageCorruption.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Unit tests for PBImageCorruptionType, CorruptionEntryBuilder and
+ * PBImageCorruption classes.
+ */
+public class TestPBImageCorruption {
+ @Test
+ public void testProperCorruptionTypeCreation() {
+ PBImageCorruption ct = new PBImageCorruption(209, false, true, 1);
+ assertEquals("CorruptNode", ct.getType());
+ ct.addMissingChildCorruption();
+ assertEquals("CorruptNodeWithMissingChild", ct.getType());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testImproperCorruptionTypeCreation() {
+ PBImageCorruption ct = new PBImageCorruption(210, false, false, 2);
+ }
+
+ @Test
+ public void testCorruptionClass() {
+ PBImageCorruption c = new PBImageCorruption(211, true, false, 3);
+ String expected = "MissingChild";
+ assertEquals(211, c.getId());
+ assertEquals(expected, c.getType());
+ assertEquals(3, c.getNumOfCorruptChildren());
+ c.addCorruptNodeCorruption();
+ expected = "CorruptNodeWithMissingChild";
+ c.setNumberOfCorruption(34);
+ assertEquals(expected, c.getType());
+ assertEquals(34, c.getNumOfCorruptChildren());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleCorruption.csv
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleCorruption.csv b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleCorruption.csv
new file mode 100644
index 0000000..e22e1dc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleCorruption.csv
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CorruptionType,Id,IsSnapshot,ParentPath,ParentId,Name,NodeType,CorruptChildren
+MissingChild,16385,false,/,Missing,,Node,1
+MissingChild,16386,false,/,16385,dir0,Node,2
+CorruptNode,16388,true,,16386,,Unknown,0
+CorruptNode,16389,true,,16386,,Unknown,0
+CorruptNodeWithMissingChild,16391,true,,16385,,Unknown,1
+CorruptNode,16394,true,,16391,,Unknown,0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleFileCorruption.csv
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleFileCorruption.csv b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleFileCorruption.csv
new file mode 100644
index 0000000..fe0ad7a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testMultipleFileCorruption.csv
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CorruptionType,Id,IsSnapshot,ParentPath,ParentId,Name,NodeType,CorruptChildren
+MissingChild,16386,false,/,16385,dir0,Node,2
+MissingChild,16391,false,/,16385,dir1,Node,1
+CorruptNode,16388,true,,16386,,Unknown,0
+CorruptNode,16389,true,,16386,,Unknown,0
+CorruptNode,16394,true,,16391,,Unknown,0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFileCorruption.csv
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFileCorruption.csv b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFileCorruption.csv
new file mode 100644
index 0000000..0f66d3a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFileCorruption.csv
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CorruptionType,Id,IsSnapshot,ParentPath,ParentId,Name,NodeType,CorruptChildren
+MissingChild,16386,false,/,16385,dir0,Node,1
+CorruptNode,16388,true,,16386,,Unknown,0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb10803d/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFolderCorruption.csv
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFolderCorruption.csv b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFolderCorruption.csv
new file mode 100644
index 0000000..0ae21de
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testSingleFolderCorruption.csv
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CorruptionType,Id,IsSnapshot,ParentPath,ParentId,Name,NodeType,CorruptChildren
+MissingChild,16385,false,/,Missing,,Node,1
+CorruptNode,16391,true,,16385,,Unknown,0
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDDS-850. ReadStateMachineData
hits OverlappingFileLockException in ContainerStateMachine. Contributed by
Shashikant Banerjee.
Posted by sh...@apache.org.
HDDS-850. ReadStateMachineData hits OverlappingFileLockException in ContainerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e102f9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e102f9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e102f9a
Branch: refs/heads/HDFS-12943
Commit: 5e102f9aa54d3057ef5f0755d45428f22a24990b
Parents: 7eb0d3a
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Thu Nov 29 22:20:08 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Thu Nov 29 22:20:08 2018 +0530
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 ++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 9 ++
.../main/proto/DatanodeContainerProtocol.proto | 1 +
.../common/src/main/resources/ozone-default.xml | 8 ++
.../server/ratis/ContainerStateMachine.java | 134 +++++++++++--------
.../server/ratis/XceiverServerRatis.java | 14 +-
.../container/keyvalue/KeyValueHandler.java | 7 +-
.../keyvalue/impl/ChunkManagerImpl.java | 11 +-
.../keyvalue/interfaces/ChunkManager.java | 5 +-
.../keyvalue/TestChunkManagerImpl.java | 6 +-
.../common/impl/TestContainerPersistence.java | 11 +-
11 files changed, 143 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 6733b8e..062b101 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -93,6 +93,14 @@ public final class ScmConfigKeys {
public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE =
"dfs.container.ratis.log.queue.size";
public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_SIZE_DEFAULT = 128;
+
+ // expiry interval stateMachineData cache entry inside containerStateMachine
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
+ "dfs.container.ratis.statemachine.cache.expiry.interval";
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
+ "10s";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.client.request.timeout.duration";
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 879f773..df233f7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -249,6 +249,15 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
+ ScmConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL;
+ public static final String
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
+ ScmConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT;
+
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 3695b6b..5237af8 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -392,6 +392,7 @@ message WriteChunkResponseProto {
message ReadChunkRequestProto {
required DatanodeBlockID blockID = 1;
required ChunkInfo chunkData = 2;
+ optional bool readFromTmpFile = 3 [default = false];
}
message ReadChunkResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aa22b2b..0545805 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -190,6 +190,14 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.statemachine.cache.expiry.interval</name>
+ <value>10s</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>The interval till which the stateMachine data in ratis
+ will be cached inside the ContainerStateMachine.
+ </description>
+ </property>
+ <property>
<name>dfs.ratis.client.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 24ecbb2..2e0c70e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
@@ -27,6 +29,7 @@ import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.impl.RaftServerConstants;
+import org.apache.ratis.server.impl.RaftServerProxy;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo;
import org.apache.ratis.thirdparty.com.google.protobuf
@@ -60,12 +63,16 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
@@ -116,12 +123,11 @@ public class ContainerStateMachine extends BaseStateMachine {
private final XceiverServerRatis ratisServer;
private final ConcurrentHashMap<Long, CompletableFuture<Message>>
writeChunkFutureMap;
- private final ConcurrentHashMap<Long, CompletableFuture<Message>>
- createContainerFutureMap;
private ExecutorService[] executors;
private final int numExecutors;
private final Map<Long, Long> applyTransactionCompletionMap;
private long lastIndex;
+ private final Cache<Long, ByteString> stateMachineDataCache;
/**
* CSM metrics.
*/
@@ -129,7 +135,7 @@ public class ContainerStateMachine extends BaseStateMachine {
public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher,
ThreadPoolExecutor chunkExecutor, XceiverServerRatis ratisServer,
- List<ExecutorService> executors) {
+ List<ExecutorService> executors, long expiryInterval) {
this.gid = gid;
this.dispatcher = dispatcher;
this.chunkExecutor = chunkExecutor;
@@ -138,9 +144,13 @@ public class ContainerStateMachine extends BaseStateMachine {
this.numExecutors = executors.size();
this.executors = executors.toArray(new ExecutorService[numExecutors]);
this.writeChunkFutureMap = new ConcurrentHashMap<>();
- this.createContainerFutureMap = new ConcurrentHashMap<>();
applyTransactionCompletionMap = new ConcurrentHashMap<>();
this.lastIndex = RaftServerConstants.INVALID_LOG_INDEX;
+ stateMachineDataCache = CacheBuilder.newBuilder()
+ .expireAfterAccess(expiryInterval, TimeUnit.MILLISECONDS)
+ // set the limit on no of cached entries equal to no of max threads
+ // executing writeStateMachineData
+ .maximumSize(chunkExecutor.getCorePoolSize()).build();
}
@Override
@@ -257,14 +267,6 @@ public class ContainerStateMachine extends BaseStateMachine {
.setStateMachineData(dataContainerCommandProto.toByteString())
.setLogData(commitContainerCommandProto.toByteString())
.build();
- } else if (proto.getCmdType() == Type.CreateContainer) {
- return TransactionContext.newBuilder()
- .setClientRequest(request)
- .setStateMachine(this)
- .setServerRole(RaftPeerRole.LEADER)
- .setStateMachineData(request.getMessage().getContent())
- .setLogData(request.getMessage().getContent())
- .build();
} else {
return TransactionContext.newBuilder()
.setClientRequest(request)
@@ -310,17 +312,17 @@ public class ContainerStateMachine extends BaseStateMachine {
private CompletableFuture<Message> handleWriteChunk(
ContainerCommandRequestProto requestProto, long entryIndex) {
final WriteChunkRequestProto write = requestProto.getWriteChunk();
- long containerID = write.getBlockID().getContainerID();
- CompletableFuture<Message> future =
- createContainerFutureMap.get(containerID);
- CompletableFuture<Message> writeChunkFuture;
- if (future != null) {
- writeChunkFuture = future.thenApplyAsync(
- v -> runCommand(requestProto), chunkExecutor);
- } else {
- writeChunkFuture = CompletableFuture.supplyAsync(
- () -> runCommand(requestProto), chunkExecutor);
+ RaftServer server = ratisServer.getServer();
+ Preconditions.checkState(server instanceof RaftServerProxy);
+ try {
+ if (((RaftServerProxy) server).getImpl(gid).isLeader()) {
+ stateMachineDataCache.put(entryIndex, write.getData());
+ }
+ } catch (IOException ioe) {
+ return completeExceptionally(ioe);
}
+ CompletableFuture<Message> writeChunkFuture = CompletableFuture
+ .supplyAsync(() -> runCommand(requestProto), chunkExecutor);
writeChunkFutureMap.put(entryIndex, writeChunkFuture);
LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID()
+ " logIndex " + entryIndex + " chunkName " + write.getChunkData()
@@ -337,14 +339,6 @@ public class ContainerStateMachine extends BaseStateMachine {
return writeChunkFuture;
}
- private CompletableFuture<Message> handleCreateContainer(
- ContainerCommandRequestProto requestProto) {
- long containerID = requestProto.getContainerID();
- createContainerFutureMap.
- computeIfAbsent(containerID, k -> new CompletableFuture<>());
- return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
- }
-
/*
* writeStateMachineData calls are not synchronized with each other
* and also with applyTransaction.
@@ -356,9 +350,10 @@ public class ContainerStateMachine extends BaseStateMachine {
final ContainerCommandRequestProto requestProto =
getRequestProto(getStateMachineData(entry.getStateMachineLogEntry()));
Type cmdType = requestProto.getCmdType();
+
+ // For only writeChunk, there will be writeStateMachineData call.
+ // CreateContainer will happen as a part of writeChunk only.
switch (cmdType) {
- case CreateContainer:
- return handleCreateContainer(requestProto);
case WriteChunk:
return handleWriteChunk(requestProto, entry.getIndex());
default:
@@ -397,7 +392,10 @@ public class ContainerStateMachine extends BaseStateMachine {
ReadChunkRequestProto.Builder readChunkRequestProto =
ReadChunkRequestProto.newBuilder()
.setBlockID(writeChunkRequestProto.getBlockID())
- .setChunkData(writeChunkRequestProto.getChunkData());
+ .setChunkData(writeChunkRequestProto.getChunkData())
+ // set readFromTempFile to true in case, the chunkFile does
+ // not exist as applyTransaction is not executed for this entry yet.
+ .setReadFromTmpFile(true);
ContainerCommandRequestProto dataContainerCommandProto =
ContainerCommandRequestProto.newBuilder(requestProto)
.setCmdType(Type.ReadChunk)
@@ -409,15 +407,39 @@ public class ContainerStateMachine extends BaseStateMachine {
dispatchCommand(dataContainerCommandProto);
ReadChunkResponseProto responseProto = response.getReadChunk();
+ ByteString data = responseProto.getData();
// assert that the response has data in it.
- Preconditions.checkNotNull(responseProto.getData());
+ Preconditions.checkNotNull(data);
+ return data;
+ }
+ /**
+ * Reads the Entry from the Cache or loads it back by reading from disk.
+ */
+ private ByteString getCachedStateMachineData(Long logIndex,
+ ContainerCommandRequestProto requestProto) throws ExecutionException {
+ try {
+ return reconstructWriteChunkRequest(
+ stateMachineDataCache.get(logIndex, new Callable<ByteString>() {
+ @Override
+ public ByteString call() throws Exception {
+ return readStateMachineData(requestProto);
+ }
+ }), requestProto);
+ } catch (ExecutionException e) {
+ throw e;
+ }
+ }
+
+ private ByteString reconstructWriteChunkRequest(ByteString data,
+ ContainerCommandRequestProto requestProto) {
+ WriteChunkRequestProto writeChunkRequestProto =
+ requestProto.getWriteChunk();
// reconstruct the write chunk request
final WriteChunkRequestProto.Builder dataWriteChunkProto =
WriteChunkRequestProto.newBuilder(writeChunkRequestProto)
// adding the state machine data
- .setData(responseProto.getData())
- .setStage(Stage.WRITE_DATA);
+ .setData(data).setStage(Stage.WRITE_DATA);
ContainerCommandRequestProto.Builder newStateMachineProto =
ContainerCommandRequestProto.newBuilder(requestProto)
@@ -455,18 +477,21 @@ public class ContainerStateMachine extends BaseStateMachine {
if (!getStateMachineData(smLogEntryProto).isEmpty()) {
return CompletableFuture.completedFuture(ByteString.EMPTY);
}
-
try {
final ContainerCommandRequestProto requestProto =
getRequestProto(entry.getStateMachineLogEntry().getLogData());
// readStateMachineData should only be called for "write" to Ratis.
Preconditions.checkArgument(!HddsUtils.isReadOnly(requestProto));
-
if (requestProto.getCmdType() == Type.WriteChunk) {
- return CompletableFuture.supplyAsync(() ->
- readStateMachineData(requestProto), chunkExecutor);
- } else if (requestProto.getCmdType() == Type.CreateContainer) {
- return CompletableFuture.completedFuture(requestProto.toByteString());
+ CompletableFuture<ByteString> future = new CompletableFuture<>();
+ return future.supplyAsync(() -> {
+ try {
+ return getCachedStateMachineData(entry.getIndex(), requestProto);
+ } catch (ExecutionException e) {
+ future.completeExceptionally(e);
+ return null;
+ }
+ }, chunkExecutor);
} else {
throw new IllegalStateException("Cmd type:" + requestProto.getCmdType()
+ " cannot have state machine data");
@@ -559,19 +584,6 @@ public class ContainerStateMachine extends BaseStateMachine {
future = CompletableFuture.supplyAsync(() -> runCommand(requestProto),
getCommandExecutor(requestProto));
}
- // Mark the createContainerFuture complete so that writeStateMachineData
- // for WriteChunk gets unblocked
- if (cmdType == Type.CreateContainer) {
- long containerID = requestProto.getContainerID();
- future.thenApply(
- r -> {
- createContainerFutureMap.remove(containerID).complete(null);
- LOG.info("create Container Transaction completed for container " +
- containerID + " log index " + index);
- return r;
- });
- }
-
lastIndex = index;
future.thenAccept(m -> {
final Long previous =
@@ -593,6 +605,11 @@ public class ContainerStateMachine extends BaseStateMachine {
return future;
}
+ private void evictStateMachineCache() {
+ stateMachineDataCache.invalidateAll();
+ stateMachineDataCache.cleanUp();
+ }
+
@Override
public void notifySlowness(RaftGroup group, RoleInfoProto roleInfoProto) {
ratisServer.handleNodeSlowness(group, roleInfoProto);
@@ -605,6 +622,13 @@ public class ContainerStateMachine extends BaseStateMachine {
}
@Override
+ public void notifyNotLeader(Collection<TransactionContext> pendingEntries)
+ throws IOException {
+ evictStateMachineCache();
+ }
+
+ @Override
public void close() throws IOException {
+ evictStateMachineCache();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index a4927e0..2675483 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -105,6 +105,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
private final StateContext context;
private final ReplicationLevel replicationLevel;
private long nodeFailureTimeoutMs;
+ private final long cacheEntryExpiryInteval;
+
private XceiverServerRatis(DatanodeDetails dd, int port,
ContainerDispatcher dispatcher, Configuration conf, StateContext context)
@@ -128,6 +130,11 @@ public final class XceiverServerRatis implements XceiverServerSpi {
conf.getEnum(OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT);
this.executors = new ArrayList<>();
+ cacheEntryExpiryInteval = conf.getTimeDuration(OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL,
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
this.dispatcher = dispatcher;
for (int i = 0; i < numContainerOpExecutors; i++) {
executors.add(Executors.newSingleThreadExecutor());
@@ -141,8 +148,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
}
private ContainerStateMachine getStateMachine(RaftGroupId gid) {
- return new ContainerStateMachine(gid, dispatcher, chunkExecutor,
- this, Collections.unmodifiableList(executors));
+ return new ContainerStateMachine(gid, dispatcher, chunkExecutor, this,
+ Collections.unmodifiableList(executors), cacheEntryExpiryInteval);
}
private RaftProperties newRaftProperties(Configuration conf) {
@@ -304,6 +311,9 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftServerConfigKeys.Log.StateMachineData.setSyncTimeoutRetry(properties,
numSyncRetries);
+ // Enable the StateMachineCaching
+ RaftServerConfigKeys.Log.StateMachineData
+ .setCachingEnabled(properties, true);
return properties;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 36c48c5..a16129e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -531,7 +531,8 @@ public class KeyValueHandler extends Handler {
.getChunkData());
Preconditions.checkNotNull(chunkInfo);
- data = chunkManager.readChunk(kvContainer, blockID, chunkInfo);
+ data = chunkManager.readChunk(kvContainer, blockID, chunkInfo,
+ request.getReadChunk().getReadFromTmpFile());
metrics.incContainerBytesStats(Type.ReadChunk, data.length);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -702,8 +703,10 @@ public class KeyValueHandler extends Handler {
ContainerProtos.ChunkInfo chunkInfo = null;
ByteString dataBuf = ByteString.EMPTY;
for (ContainerProtos.ChunkInfo chunk : responseData.getChunks()) {
+ // if the block is committed, all chunks must have been committed.
+ // Tmp chunk files won't exist here.
byte[] data = chunkManager.readChunk(kvContainer, blockID,
- ChunkInfo.getFromProtoBuf(chunk));
+ ChunkInfo.getFromProtoBuf(chunk), false);
ByteString current = ByteString.copyFrom(data);
dataBuf = dataBuf.concat(current);
chunkInfo = chunk;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index a1c3d01..e41346c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -173,13 +173,14 @@ public class ChunkManagerImpl implements ChunkManager {
* @param container - Container for the chunk
* @param blockID - ID of the block.
* @param info - ChunkInfo.
+ * @param readFromTmpFile whether to read from tmp chunk file or not.
* @return byte array
* @throws StorageContainerException
* TODO: Right now we do not support partial reads and writes of chunks.
* TODO: Explore if we need to do that for ozone.
*/
- public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info)
- throws StorageContainerException {
+ public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
+ boolean readFromTmpFile) throws StorageContainerException {
try {
KeyValueContainerData containerData = (KeyValueContainerData) container
.getContainerData();
@@ -194,6 +195,12 @@ public class ChunkManagerImpl implements ChunkManager {
if (containerData.getLayOutVersion() == ChunkLayOutVersion
.getLatestVersion().getVersion()) {
File chunkFile = ChunkUtils.getChunkFile(containerData, info);
+
+ // In case the chunk file does not exist but tmp chunk file exist,
+ // read from tmp chunk file if readFromTmpFile is set to true
+ if (!chunkFile.exists() && readFromTmpFile) {
+ chunkFile = getTmpChunkFile(chunkFile, info);
+ }
data = ChunkUtils.readData(chunkFile, info, volumeIOStats);
containerData.incrReadCount();
long length = chunkFile.length();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index 3b06585..f90346e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -51,14 +51,15 @@ public interface ChunkManager {
* @param container - Container for the chunk
* @param blockID - ID of the block.
* @param info - ChunkInfo.
+ * @param readFromTmpFile whether to read from tmp chunk file or not
* @return byte array
* @throws StorageContainerException
*
* TODO: Right now we do not support partial reads and writes of chunks.
* TODO: Explore if we need to do that for ozone.
*/
- byte[] readChunk(Container container, BlockID blockID, ChunkInfo info) throws
- StorageContainerException;
+ byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
+ boolean readFromTmpFile) throws StorageContainerException;
/**
* Deletes a given chunk.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index 717dd05..ef77204 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -189,7 +189,7 @@ public class TestChunkManagerImpl {
checkWriteIOStats(data.length, 1);
checkReadIOStats(0, 0);
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
- chunkInfo);
+ chunkInfo, false);
assertEquals(expectedData.length, data.length);
assertTrue(Arrays.equals(expectedData, data));
checkReadIOStats(data.length, 1);
@@ -226,7 +226,7 @@ public class TestChunkManagerImpl {
try {
// trying to read a chunk, where chunk file does not exist
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
- chunkInfo);
+ chunkInfo, false);
fail("testReadChunkFileNotExists failed");
} catch (StorageContainerException ex) {
GenericTestUtils.assertExceptionContains("Unable to find the chunk " +
@@ -249,7 +249,7 @@ public class TestChunkManagerImpl {
for (int i=0; i<100; i++) {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), i), 0, data.length);
- chunkManager.readChunk(keyValueContainer, blockID, chunkInfo);
+ chunkManager.readChunk(keyValueContainer, blockID, chunkInfo, false);
}
checkReadIOStats(data.length*100, 100);
assertTrue(hddsVolume.getVolumeIOStats().getReadTime() > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e102f9a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index fb7b0c5..5d19a10 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -406,7 +406,7 @@ public class TestContainerPersistence {
for (int x = 0; x < chunkCount; x++) {
String fileName = String.format("%s.data.%d", blockID.getLocalID(), x);
ChunkInfo info = fileHashMap.get(fileName);
- byte[] data = chunkManager.readChunk(container, blockID, info);
+ byte[] data = chunkManager.readChunk(container, blockID, info, false);
ChecksumData checksumData = checksum.computeChecksum(data);
Assert.assertEquals(info.getChecksumData(), checksumData);
}
@@ -435,11 +435,11 @@ public class TestContainerPersistence {
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
COMBINED);
- byte[] readData = chunkManager.readChunk(container, blockID, info);
+ byte[] readData = chunkManager.readChunk(container, blockID, info, false);
assertTrue(Arrays.equals(data, readData));
ChunkInfo info2 = getChunk(blockID.getLocalID(), 0, start, length);
- byte[] readData2 = chunkManager.readChunk(container, blockID, info2);
+ byte[] readData2 = chunkManager.readChunk(container, blockID, info2, false);
assertEquals(length, readData2.length);
assertTrue(Arrays.equals(
Arrays.copyOfRange(data, start, start + length), readData2));
@@ -513,7 +513,8 @@ public class TestContainerPersistence {
// Request to read the whole data in a single go.
ChunkInfo largeChunk = getChunk(blockID.getLocalID(), 0, 0,
datalen * chunkCount);
- byte[] newdata = chunkManager.readChunk(container, blockID, largeChunk);
+ byte[] newdata =
+ chunkManager.readChunk(container, blockID, largeChunk, false);
MessageDigest newSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
newSha.update(newdata);
Assert.assertEquals(Hex.encodeHexString(oldSha.digest()),
@@ -543,7 +544,7 @@ public class TestContainerPersistence {
chunkManager.deleteChunk(container, blockID, info);
exception.expect(StorageContainerException.class);
exception.expectMessage("Unable to find the chunk file.");
- chunkManager.readChunk(container, blockID, info);
+ chunkManager.readChunk(container, blockID, info, false);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HADOOP-15950. Failover for
LdapGroupsMapping. Contributed by Lukas Majercak.
Posted by sh...@apache.org.
HADOOP-15950. Failover for LdapGroupsMapping. Contributed by Lukas Majercak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9a3aa64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9a3aa64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9a3aa64
Branch: refs/heads/HDFS-12943
Commit: c9a3aa64dc95b097c51070f71a3b1a2ad126b2b9
Parents: ef3b03b
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Mon Dec 3 12:10:05 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Mon Dec 3 12:10:05 2018 -0800
----------------------------------------------------------------------
.../hadoop/security/LdapGroupsMapping.java | 163 +++++++++++++------
.../src/main/resources/core-default.xml | 28 +++-
.../src/site/markdown/GroupsMapping.md | 54 +++++-
.../hadoop/security/TestLdapGroupsMapping.java | 80 ++++-----
.../security/TestLdapGroupsMappingBase.java | 76 ++++++++-
.../TestLdapGroupsMappingWithFailover.java | 142 ++++++++++++++++
.../TestLdapGroupsMappingWithOneQuery.java | 16 +-
.../TestLdapGroupsMappingWithPosixGroup.java | 10 +-
8 files changed, 450 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 6beaa9e..83eb5ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Hashtable;
+import java.util.Iterator;
import java.util.List;
import java.util.HashSet;
import java.util.Collection;
@@ -40,7 +41,10 @@ import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import javax.naming.ldap.LdapName;
import javax.naming.ldap.Rdn;
+import javax.naming.spi.InitialContextFactory;
+import com.google.common.collect.Iterators;
+import com.sun.jndi.ldap.LdapCtxFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
@@ -83,7 +87,7 @@ public class LdapGroupsMapping
public static final String LDAP_CONFIG_PREFIX = "hadoop.security.group.mapping.ldap";
/*
- * URL of the LDAP server
+ * URL of the LDAP server(s)
*/
public static final String LDAP_URL_KEY = LDAP_CONFIG_PREFIX + ".url";
public static final String LDAP_URL_DEFAULT = "";
@@ -232,6 +236,20 @@ public class LdapGroupsMapping
LDAP_CONFIG_PREFIX + ".read.timeout.ms";
public static final int READ_TIMEOUT_DEFAULT = 60 * 1000; // 60 seconds
+ public static final String LDAP_NUM_ATTEMPTS_KEY =
+ LDAP_CONFIG_PREFIX + ".num.attempts";
+ public static final int LDAP_NUM_ATTEMPTS_DEFAULT = 3;
+
+ public static final String LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY =
+ LDAP_CONFIG_PREFIX + ".num.attempts.before.failover";
+ public static final int LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT =
+ LDAP_NUM_ATTEMPTS_DEFAULT;
+
+ public static final String LDAP_CTX_FACTORY_CLASS_KEY =
+ LDAP_CONFIG_PREFIX + ".ctx.factory.class";
+ public static final Class<? extends LdapCtxFactory>
+ LDAP_CTX_FACTORY_CLASS_DEFAULT = LdapCtxFactory.class;
+
private static final Logger LOG =
LoggerFactory.getLogger(LdapGroupsMapping.class);
@@ -242,8 +260,10 @@ public class LdapGroupsMapping
private DirContext ctx;
private Configuration conf;
-
- private String ldapUrl;
+
+ private Iterator<String> ldapUrls;
+ private String currentLdapUrl;
+
private boolean useSsl;
private String keystore;
private String keystorePass;
@@ -258,14 +278,15 @@ public class LdapGroupsMapping
private String memberOfAttr;
private String groupMemberAttr;
private String groupNameAttr;
- private int groupHierarchyLevels;
+ private int groupHierarchyLevels;
private String posixUidAttr;
private String posixGidAttr;
private boolean isPosix;
private boolean useOneQuery;
+ private int numAttempts;
+ private int numAttemptsBeforeFailover;
+ private Class<? extends InitialContextFactory> ldapCxtFactoryClass;
- public static final int RECONNECT_RETRY_COUNT = 3;
-
/**
* Returns list of groups for a user.
*
@@ -279,20 +300,31 @@ public class LdapGroupsMapping
@Override
public synchronized List<String> getGroups(String user) {
/*
- * Normal garbage collection takes care of removing Context instances when they are no longer in use.
- * Connections used by Context instances being garbage collected will be closed automatically.
- * So in case connection is closed and gets CommunicationException, retry some times with new new DirContext/connection.
+ * Normal garbage collection takes care of removing Context instances when
+ * they are no longer in use. Connections used by Context instances being
+ * garbage collected will be closed automatically. So in case connection is
+ * closed and gets CommunicationException, retry some times with new new
+ * DirContext/connection.
*/
- for(int retry = 0; retry < RECONNECT_RETRY_COUNT; retry++) {
+
+ // Tracks the number of attempts made using the same LDAP server
+ int atemptsBeforeFailover = 1;
+
+ for (int attempt = 1; attempt <= numAttempts; attempt++,
+ atemptsBeforeFailover++) {
try {
return doGetGroups(user, groupHierarchyLevels);
} catch (NamingException e) {
- LOG.warn("Failed to get groups for user " + user + " (retry=" + retry
- + ") by " + e);
+ LOG.warn("Failed to get groups for user {} (attempt={}/{}) using {}. " +
+ "Exception: ", user, attempt, numAttempts, currentLdapUrl, e);
LOG.trace("TRACE", e);
+
+ if (failover(atemptsBeforeFailover, numAttemptsBeforeFailover)) {
+ atemptsBeforeFailover = 0;
+ }
}
- //reset ctx so that new DirContext can be created with new connection
+ // Reset ctx so that new DirContext can be created with new connection
this.ctx = null;
}
@@ -378,10 +410,10 @@ public class LdapGroupsMapping
private List<String> lookupGroup(SearchResult result, DirContext c,
int goUpHierarchy)
throws NamingException {
- List<String> groups = new ArrayList<String>();
- Set<String> groupDNs = new HashSet<String>();
+ List<String> groups = new ArrayList<>();
+ Set<String> groupDNs = new HashSet<>();
- NamingEnumeration<SearchResult> groupResults = null;
+ NamingEnumeration<SearchResult> groupResults;
// perform the second LDAP query
if (isPosix) {
groupResults = lookupPosixGroup(result, c);
@@ -402,10 +434,10 @@ public class LdapGroupsMapping
}
if (goUpHierarchy > 0 && !isPosix) {
// convert groups to a set to ensure uniqueness
- Set<String> groupset = new HashSet<String>(groups);
+ Set<String> groupset = new HashSet<>(groups);
goUpGroupHierarchy(groupDNs, goUpHierarchy, groupset);
// convert set back to list for compatibility
- groups = new ArrayList<String>(groupset);
+ groups = new ArrayList<>(groupset);
}
}
return groups;
@@ -433,11 +465,9 @@ public class LdapGroupsMapping
userSearchFilter, new Object[]{user}, SEARCH_CONTROLS);
// return empty list if the user can not be found.
if (!results.hasMoreElements()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("doGetGroups(" + user + ") returned no groups because the " +
- "user is not found.");
- }
- return new ArrayList<String>();
+ LOG.debug("doGetGroups({}) returned no groups because the " +
+ "user is not found.", user);
+ return new ArrayList<>();
}
SearchResult result = results.nextElement();
@@ -455,7 +485,7 @@ public class LdapGroupsMapping
memberOfAttr + "' attribute." +
"Returned user object: " + result.toString());
}
- groups = new ArrayList<String>();
+ groups = new ArrayList<>();
NamingEnumeration groupEnumeration = groupDNAttr.getAll();
while (groupEnumeration.hasMore()) {
String groupDN = groupEnumeration.next().toString();
@@ -470,9 +500,7 @@ public class LdapGroupsMapping
if (groups == null || groups.isEmpty() || goUpHierarchy > 0) {
groups = lookupGroup(result, c, goUpHierarchy);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("doGetGroups(" + user + ") returned " + groups);
- }
+ LOG.debug("doGetGroups({}) returned {}", user, groups);
return groups;
}
@@ -480,7 +508,7 @@ public class LdapGroupsMapping
*/
void getGroupNames(SearchResult groupResult, Collection<String> groups,
Collection<String> groupDNs, boolean doGetDNs)
- throws NamingException {
+ throws NamingException {
Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
if (groupName == null) {
throw new NamingException("The group object does not have " +
@@ -517,7 +545,7 @@ public class LdapGroupsMapping
return;
}
DirContext context = getDirContext();
- Set<String> nextLevelGroups = new HashSet<String>();
+ Set<String> nextLevelGroups = new HashSet<>();
StringBuilder filter = new StringBuilder();
filter.append("(&").append(groupSearchFilter).append("(|");
for (String dn : groupDNs) {
@@ -537,13 +565,32 @@ public class LdapGroupsMapping
goUpGroupHierarchy(nextLevelGroups, goUpHierarchy - 1, groups);
}
- DirContext getDirContext() throws NamingException {
+ /**
+ * Check whether we should fail over to the next LDAP server.
+ * @param attemptsMadeWithSameLdap current number of attempts made
+ * with using same LDAP instance
+ * @param maxAttemptsBeforeFailover maximum number of attempts
+ * before failing over
+ * @return true if we should fail over to the next LDAP server
+ */
+ protected boolean failover(
+ int attemptsMadeWithSameLdap, int maxAttemptsBeforeFailover) {
+ if (attemptsMadeWithSameLdap >= maxAttemptsBeforeFailover) {
+ String previousLdapUrl = currentLdapUrl;
+ currentLdapUrl = ldapUrls.next();
+ LOG.info("Reached {} attempts on {}, failing over to {}",
+ attemptsMadeWithSameLdap, previousLdapUrl, currentLdapUrl);
+ return true;
+ }
+ return false;
+ }
+
+ private DirContext getDirContext() throws NamingException {
if (ctx == null) {
// Set up the initial environment for LDAP connectivity
- Hashtable<String, String> env = new Hashtable<String, String>();
- env.put(Context.INITIAL_CONTEXT_FACTORY,
- com.sun.jndi.ldap.LdapCtxFactory.class.getName());
- env.put(Context.PROVIDER_URL, ldapUrl);
+ Hashtable<String, String> env = new Hashtable<>();
+ env.put(Context.INITIAL_CONTEXT_FACTORY, ldapCxtFactoryClass.getName());
+ env.put(Context.PROVIDER_URL, currentLdapUrl);
env.put(Context.SECURITY_AUTHENTICATION, "simple");
// Set up SSL security, if necessary
@@ -581,7 +628,7 @@ public class LdapGroupsMapping
* Caches groups, no need to do that for this provider
*/
@Override
- public void cacheGroupsRefresh() throws IOException {
+ public void cacheGroupsRefresh() {
// does nothing in this provider of user to groups mapping
}
@@ -591,7 +638,7 @@ public class LdapGroupsMapping
* @param groups unused
*/
@Override
- public void cacheGroupsAdd(List<String> groups) throws IOException {
+ public void cacheGroupsAdd(List<String> groups) {
// does nothing in this provider of user to groups mapping
}
@@ -602,10 +649,12 @@ public class LdapGroupsMapping
@Override
public synchronized void setConf(Configuration conf) {
- ldapUrl = conf.get(LDAP_URL_KEY, LDAP_URL_DEFAULT);
- if (ldapUrl == null || ldapUrl.isEmpty()) {
- throw new RuntimeException("LDAP URL is not configured");
+ String[] urls = conf.getStrings(LDAP_URL_KEY, LDAP_URL_DEFAULT);
+ if (urls == null || urls.length == 0) {
+ throw new RuntimeException("LDAP URL(s) are not configured");
}
+ ldapUrls = Iterators.cycle(urls);
+ currentLdapUrl = ldapUrls.next();
useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT);
if (useSsl) {
@@ -621,17 +670,13 @@ public class LdapGroupsMapping
String baseDN = conf.getTrimmed(BASE_DN_KEY, BASE_DN_DEFAULT);
- //User search base which defaults to base dn.
+ // User search base which defaults to base dn.
userbaseDN = conf.getTrimmed(USER_BASE_DN_KEY, baseDN);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Usersearch baseDN: " + userbaseDN);
- }
+ LOG.debug("Usersearch baseDN: {}", userbaseDN);
- //Group search base which defaults to base dn.
+ // Group search base which defaults to base dn.
groupbaseDN = conf.getTrimmed(GROUP_BASE_DN_KEY, baseDN);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Groupsearch baseDN: " + userbaseDN);
- }
+ LOG.debug("Groupsearch baseDN: {}", groupbaseDN);
groupSearchFilter =
conf.get(GROUP_SEARCH_FILTER_KEY, GROUP_SEARCH_FILTER_DEFAULT);
@@ -655,7 +700,8 @@ public class LdapGroupsMapping
posixGidAttr =
conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT);
- int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
+ int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT,
+ DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout);
// Limit the attributes returned to only those required to speed up the search.
// See HADOOP-10626 and HADOOP-12001 for more details.
@@ -669,9 +715,26 @@ public class LdapGroupsMapping
}
SEARCH_CONTROLS.setReturningAttributes(returningAttributes);
+ ldapCxtFactoryClass = conf.getClass(LDAP_CTX_FACTORY_CLASS_KEY,
+ LDAP_CTX_FACTORY_CLASS_DEFAULT, InitialContextFactory.class);
+
+ this.numAttempts = conf.getInt(LDAP_NUM_ATTEMPTS_KEY,
+ LDAP_NUM_ATTEMPTS_DEFAULT);
+ this.numAttemptsBeforeFailover = conf.getInt(
+ LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
+ LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT);
+
this.conf = conf;
}
+ /**
+ * Get URLs of configured LDAP servers.
+ * @return URLs of LDAP servers being used.
+ */
+ public Iterator<String> getLdapUrls() {
+ return ldapUrls;
+ }
+
private void loadSslConf(Configuration sslConf) {
keystore = sslConf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT);
keystorePass = getPassword(sslConf, LDAP_KEYSTORE_PASSWORD_KEY,
@@ -721,8 +784,8 @@ public class LdapGroupsMapping
password = new String(passchars);
}
} catch (IOException ioe) {
- LOG.warn("Exception while trying to get password for alias " + alias
- + ": ", ioe);
+ LOG.warn("Exception while trying to get password for alias {}:",
+ alias, ioe);
}
return password;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b243a9c..28e910b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -225,11 +225,35 @@
</property>
<property>
+ <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+ <value>3</value>
+ <description>
+ This property is the number of attempts to be made for LDAP operations.
+ If this limit is exceeded, LdapGroupsMapping will return an empty
+ group list.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
+ <value>3</value>
+ <description>
+ This property is the number of attempts to be made for LDAP operations
+ using a single LDAP instance. If multiple LDAP servers are configured
+ and this number of failed operations is reached, we will switch to the
+ next LDAP server. The configuration for the overall number of attempts
+ will still be respected, failover will thus be performed only if this
+ property is less than hadoop.security.group.mapping.ldap.num.attempts.
+ </description>
+</property>
+
+<property>
<name>hadoop.security.group.mapping.ldap.url</name>
<value></value>
<description>
- The URL of the LDAP server to use for resolving user groups when using
- the LdapGroupsMapping user to group mapping.
+ The URL of the LDAP server(s) to use for resolving user groups when using
+ the LdapGroupsMapping user to group mapping. Supports configuring multiple
+ LDAP servers via a comma-separated list.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index c6af930..b957821 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -73,7 +73,7 @@ The cache invalidation is configurable via `hadoop.security.groups.negative-cach
LDAP Groups Mapping
--------
This provider supports LDAP with simple password authentication using JNDI API.
-`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server for resolving user groups.
+`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server(s) for resolving user groups. It supports configuring multiple LDAP servers via a comma-separated list.
`hadoop.security.group.mapping.ldap.base` configures the search base for the LDAP connection. This is a distinguished name, and will typically be the root of the LDAP directory.
Get groups for a given username first looks up the user and then looks up the groups for the user result. If the directory setup has different user and group search bases, use `hadoop.security.group.mapping.ldap.userbase` and `hadoop.security.group.mapping.ldap.groupbase` configs.
@@ -115,6 +115,58 @@ To enable this feature, set `hadoop.security.group.mapping.ldap.search.attr.memb
If the LDAP server's certificate is not signed by a well known certificate authority, specify the path to the truststore in `hadoop.security.group.mapping.ldap.ssl.truststore`.
Similar to keystore, specify the truststore password file in `hadoop.security.group.mapping.ldap.ssl.truststore.password.file`.
+### Configuring retries and multiple LDAP servers with failover ###
+If there are issues encountered when retrieving information from LDAP servers, the request will be retried. To configure the number of retries, use the following configuration:
+
+```<property>
+ <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+ <value>3</value>
+ <description>
+ This property is the number of attempts to be made for LDAP operations.
+ If this limit is exceeded, LdapGroupsMapping will return an empty
+ group list.
+ </description>
+ </property>
+```
+
+LDAP Groups Mapping also supports configuring multiple LDAP servers and failover if a particular instance is not available or is misbehaving.
+The following configuration shows configuring 3 LDAP servers. Additionally, 2 attempts will be made for each server before failing over to the next one, with 6 attempts overall before failing.
+
+```
+<property>
+ <name>hadoop.security.group.mapping.ldap.url</name>
+ <value>ldap://server1,ldap://server2,ldap://server3</value>
+ <description>
+ The URL of the LDAP server(s) to use for resolving user groups when using
+ the LdapGroupsMapping user to group mapping. Supports configuring multiple
+ LDAP servers via a comma-separated list.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+ <value>6</value>
+ <description>
+ This property is the number of attempts to be made for LDAP operations.
+ If this limit is exceeded, LdapGroupsMapping will return an empty
+ group list.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
+ <value>2</value>
+ <description>
+ This property is the number of attempts to be made for LDAP operations
+ using a single LDAP instance. If multiple LDAP servers are configured
+ and this number of failed operations is reached, we will switch to the
+ next LDAP server. The configuration for the overall number of attempts
+ will still be respected, failover will thus be performed only if this
+ property is less than hadoop.security.group.mapping.ldap.num.attempts.
+ </description>
+</property>
+```
+
Composite Groups Mapping
--------
`CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
index d9f0c2a..a361d04 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
@@ -18,11 +18,12 @@
package org.apache.hadoop.security;
import static org.apache.hadoop.security.LdapGroupsMapping.CONNECTION_TIMEOUT;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
import static org.apache.hadoop.security.LdapGroupsMapping.READ_TIMEOUT;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
@@ -82,14 +83,16 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
private final String userDN = "CN=some_user,DC=test,DC=com";
+ private static final String TEST_LDAP_URL = "ldap://test";
+
@Before
- public void setupMocks() throws NamingException {
+ public void setupMocks() {
when(getUserSearchResult().getNameInNamespace()).
thenReturn(userDN);
}
@Test
- public void testGetGroups() throws IOException, NamingException {
+ public void testGetGroups() throws NamingException {
// The search functionality of the mock context is reused, so we will
// return the user NamingEnumeration first, and then the group
when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -100,9 +103,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
@Test
public void testGetGroupsWithDifferentBaseDNs() throws Exception {
- Configuration conf = new Configuration();
- // Set this, so we don't throw an exception
- conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+ Configuration conf = getBaseConf(TEST_LDAP_URL);
String userBaseDN = "ou=Users,dc=xxx,dc=com ";
String groupBaseDN = " ou=Groups,dc=xxx,dc=com";
conf.set(LdapGroupsMapping.USER_BASE_DN_KEY, userBaseDN);
@@ -113,9 +114,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
@Test
public void testGetGroupsWithDefaultBaseDN() throws Exception {
- Configuration conf = new Configuration();
- // Set this, so we don't throw an exception
- conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+ Configuration conf = getBaseConf(TEST_LDAP_URL);
String baseDN = " dc=xxx,dc=com ";
conf.set(LdapGroupsMapping.BASE_DN_KEY, baseDN);
doTestGetGroupsWithBaseDN(conf, baseDN.trim(), baseDN.trim());
@@ -159,7 +158,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
}
@Test
- public void testGetGroupsWithHierarchy() throws IOException, NamingException {
+ public void testGetGroupsWithHierarchy() throws NamingException {
// The search functionality of the mock context is reused, so we will
// return the user NamingEnumeration first, and then the group
// The parent search is run once for each level, and is a different search
@@ -175,7 +174,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
}
@Test
- public void testGetGroupsWithConnectionClosed() throws IOException, NamingException {
+ public void testGetGroupsWithConnectionClosed() throws NamingException {
// The case mocks connection is closed/gc-ed, so the first search call throws CommunicationException,
// then after reconnected return the user NamingEnumeration first, and then the group
when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -190,21 +189,20 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
}
@Test
- public void testGetGroupsWithLdapDown() throws IOException, NamingException {
+ public void testGetGroupsWithLdapDown() throws NamingException {
// This mocks the case where Ldap server is down, and always throws CommunicationException
when(getContext().search(anyString(), anyString(), any(Object[].class),
any(SearchControls.class)))
.thenThrow(new CommunicationException("Connection is closed"));
// Ldap server is down, no groups should be retrieved
- doTestGetGroups(Arrays.asList(new String[] {}),
- LdapGroupsMapping.RECONNECT_RETRY_COUNT);
+ doTestGetGroups(Arrays.asList(new String[] {}), 4);
}
- private void doTestGetGroups(List<String> expectedGroups, int searchTimes) throws IOException, NamingException {
- Configuration conf = new Configuration();
- // Set this, so we don't throw an exception
- conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+ private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
+ throws NamingException {
+ Configuration conf = getBaseConf(TEST_LDAP_URL);
+ conf.setInt(LDAP_NUM_ATTEMPTS_KEY, searchTimes);
LdapGroupsMapping groupsMapping = getGroupsMapping();
groupsMapping.setConf(conf);
@@ -216,17 +214,12 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
// We should have searched for a user, and then two groups
verify(getContext(), times(searchTimes)).search(anyString(),
- anyString(),
- any(Object[].class),
- any(SearchControls.class));
+ anyString(), any(Object[].class), any(SearchControls.class));
}
private void doTestGetGroupsWithParent(List<String> expectedGroups,
- int searchTimesGroup, int searchTimesParentGroup)
- throws IOException, NamingException {
- Configuration conf = new Configuration();
- // Set this, so we don't throw an exception
- conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+ int searchTimesGroup, int searchTimesParentGroup) throws NamingException {
+ Configuration conf = getBaseConf(TEST_LDAP_URL);
// Set the config to get parents 1 level up
conf.setInt(LdapGroupsMapping.GROUP_HIERARCHY_LEVELS_KEY, 1);
@@ -237,18 +230,14 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
List<String> groups = groupsMapping.getGroups("some_user");
// compare lists, ignoring the order
- Assert.assertEquals(new HashSet<String>(expectedGroups),
- new HashSet<String>(groups));
+ Assert.assertEquals(new HashSet<>(expectedGroups), new HashSet<>(groups));
// We should have searched for a user, and group
verify(getContext(), times(searchTimesGroup)).search(anyString(),
- anyString(),
- any(Object[].class),
- any(SearchControls.class));
+ anyString(), any(Object[].class), any(SearchControls.class));
// One groups search for the parent group should have been done
- verify(getContext(), times(searchTimesParentGroup)).search(anyString(),
- anyString(),
- any(SearchControls.class));
+ verify(getContext(), times(searchTimesParentGroup)).search(
+ anyString(), anyString(), any(SearchControls.class));
}
@Test
@@ -268,7 +257,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
@Test
public void testConfGetPassword() throws Exception {
File testDir = GenericTestUtils.getTestDir();
- Configuration conf = new Configuration();
+ Configuration conf = getBaseConf();
final Path jksPath = new Path(testDir.toString(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
@@ -283,10 +272,10 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'};
// ensure that we get nulls when the key isn't there
- assertEquals(null, provider.getCredentialEntry(
+ assertNull(provider.getCredentialEntry(
LdapGroupsMapping.BIND_PASSWORD_KEY));
- assertEquals(null, provider.getCredentialEntry
- (LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
+ assertNull(provider.getCredentialEntry(
+ LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
// create new aliases
try {
@@ -350,9 +339,8 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
ldapServer.start();
final LdapGroupsMapping mapping = new LdapGroupsMapping();
- final Configuration conf = new Configuration();
- conf.set(LdapGroupsMapping.LDAP_URL_KEY,
- "ldap://localhost:" + serverSock.getLocalPort());
+ String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
+ final Configuration conf = getBaseConf(ldapUrl, null);
conf.setInt(CONNECTION_TIMEOUT, connectionTimeoutMs);
mapping.setConf(conf);
@@ -406,9 +394,8 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
ldapServer.start();
final LdapGroupsMapping mapping = new LdapGroupsMapping();
- final Configuration conf = new Configuration();
- conf.set(LdapGroupsMapping.LDAP_URL_KEY,
- "ldap://localhost:" + serverSock.getLocalPort());
+ String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
+ final Configuration conf = getBaseConf(ldapUrl, null);
conf.setInt(READ_TIMEOUT, readTimeoutMs);
mapping.setConf(conf);
@@ -436,15 +423,12 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
*/
@Test(timeout = 10000)
public void testSetConf() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getBaseConf(TEST_LDAP_URL);
Configuration mockConf = Mockito.spy(conf);
when(mockConf.getPassword(anyString()))
.thenThrow(new IOException("injected IOException"));
- // Set a dummy LDAP server URL.
- mockConf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
LdapGroupsMapping groupsMapping = getGroupsMapping();
groupsMapping.setConf(mockConf);
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
index 51d3673..2469b4b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
@@ -18,10 +18,15 @@
package org.apache.hadoop.security;
-import static org.mockito.Mockito.doReturn;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_DEFAULT;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import javax.naming.Context;
import javax.naming.NamingEnumeration;
import javax.naming.NamingException;
import javax.naming.directory.Attribute;
@@ -31,13 +36,17 @@ import javax.naming.directory.BasicAttributes;
import javax.naming.directory.DirContext;
import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
+import javax.naming.spi.InitialContextFactory;
+import org.apache.hadoop.conf.Configuration;
import org.junit.Before;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
+import java.util.Hashtable;
+
public class TestLdapGroupsMappingBase {
@Mock
private DirContext context;
@@ -62,7 +71,6 @@ public class TestLdapGroupsMappingBase {
public void setupMocksBase() throws NamingException {
MockitoAnnotations.initMocks(this);
DirContext ctx = getContext();
- doReturn(ctx).when(groupsMapping).getDirContext();
when(ctx.search(Mockito.anyString(), Mockito.anyString(),
Mockito.any(Object[].class), Mockito.any(SearchControls.class))).
@@ -117,9 +125,30 @@ public class TestLdapGroupsMappingBase {
thenReturn("CN=some_group,DC=test,DC=com");
}
+ protected Configuration getBaseConf() {
+ return getBaseConf("ldap://test");
+ }
+
+ protected Configuration getBaseConf(String ldapUrl) {
+ return getBaseConf(ldapUrl, getContext());
+ }
+
+ protected Configuration getBaseConf(
+ String ldapUrl, DirContext contextToReturn) {
+ DummyLdapCtxFactory.setContextToReturn(contextToReturn);
+ DummyLdapCtxFactory.setExpectedLdapUrl(ldapUrl);
+
+ Configuration conf = new Configuration();
+ conf.set(LDAP_URL_KEY, ldapUrl);
+ conf.setClass(LDAP_CTX_FACTORY_CLASS_KEY, DummyLdapCtxFactory.class,
+ InitialContextFactory.class);
+ return conf;
+ }
+
protected DirContext getContext() {
return context;
}
+
protected NamingEnumeration<SearchResult> getUserNames() {
return userNames;
}
@@ -139,13 +168,56 @@ public class TestLdapGroupsMappingBase {
protected LdapGroupsMapping getGroupsMapping() {
return groupsMapping;
}
+
protected String[] getTestGroups() {
return testGroups;
}
+
protected NamingEnumeration getParentGroupNames() {
return parentGroupNames;
}
+
protected String[] getTestParentGroups() {
return testParentGroups;
}
+
+ /**
+ * Ldap Context Factory implementation to be used for testing to check
+ * contexts are requested for the expected LDAP server URLs etc.
+ */
+ public static class DummyLdapCtxFactory implements InitialContextFactory {
+
+ private static DirContext contextToReturn;
+ private static String expectedLdapUrl;
+
+ public DummyLdapCtxFactory() {
+ }
+
+ protected static void setContextToReturn(DirContext ctx) {
+ contextToReturn = ctx;
+ }
+
+ protected static void setExpectedLdapUrl(String url) {
+ expectedLdapUrl = url;
+ }
+
+ @Override
+ public Context getInitialContext(Hashtable<?, ?> env)
+ throws NamingException {
+ if (expectedLdapUrl != null) {
+ String actualLdapUrl = (String) env.get(Context.PROVIDER_URL);
+ assertEquals(expectedLdapUrl, actualLdapUrl);
+ }
+ if (contextToReturn == null) {
+ InitialContextFactory defaultFactory = null;
+ try {
+ defaultFactory = LDAP_CTX_FACTORY_CLASS_DEFAULT.newInstance();
+ } catch (ReflectiveOperationException e) {
+ fail("Could not initialize the default factory");
+ }
+ return defaultFactory.getInitialContext(env);
+ }
+ return contextToReturn;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
new file mode 100644
index 0000000..4f6ea1f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import javax.naming.CommunicationException;
+import javax.naming.directory.SearchControls;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test failover functionality for switching to different
+ * LDAP server URLs upon failures.
+ */
+public class TestLdapGroupsMappingWithFailover
+ extends TestLdapGroupsMappingBase {
+
+ private static final String TEST_USER_NAME = "some_user";
+
+ /**
+ * Test that when disabled, we will retry the configured number
+ * of times using the same LDAP server.
+ */
+ @Test
+ public void testDoesNotFailoverWhenDisabled() throws Exception {
+ final int numAttempts = 3;
+ Configuration conf = getBaseConf();
+ conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
+ "ldap://test2");
+ DummyLdapCtxFactory.setExpectedLdapUrl("ldap://test");
+ conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
+ conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY, numAttempts);
+
+ when(getContext().search(anyString(), anyString(), any(Object[].class),
+ any(SearchControls.class))).thenThrow(new CommunicationException());
+
+ LdapGroupsMapping groupsMapping = getGroupsMapping();
+ groupsMapping.setConf(conf);
+
+ List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
+ assertTrue(groups.isEmpty());
+
+ // Test that we made 3 attempts using the same server
+ verify(getContext(), times(numAttempts)).search(anyString(), anyString(),
+ any(Object[].class), any(SearchControls.class));
+ }
+
+ /**
+ * Test that when configured, we will make the specified amount of
+ * attempts using one ldap url before failing over to the next one.
+ *
+ * This also tests that we wrap back to the first server
+ * if we've tried them all.
+ */
+ @Test
+ public void testFailover() throws Exception {
+ Queue<String> ldapUrls = new LinkedList<>();
+ ldapUrls.add("ldap://test");
+ ldapUrls.add("ldap://test1");
+ ldapUrls.add("ldap://test2");
+
+ final int numAttempts = 12;
+ final int numAttemptsBeforeFailover = 2;
+
+ Configuration conf = getBaseConf();
+ conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
+ "ldap://test2");
+ conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
+ conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
+ numAttemptsBeforeFailover);
+
+ // Set the first expected url and add it back to the queue
+ String nextLdapUrl = ldapUrls.remove();
+ DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
+ ldapUrls.add(nextLdapUrl);
+
+ // Number of attempts using a single ldap server url
+ final AtomicInteger serverAttempts = new AtomicInteger(
+ numAttemptsBeforeFailover);
+
+ when(getContext().search(anyString(), anyString(), any(Object[].class),
+ any(SearchControls.class))).thenAnswer(new Answer<Object>() {
+ @Override
+ public Object answer(InvocationOnMock invocationOnMock)
+ throws Throwable {
+ if (serverAttempts.get() == 1) {
+ // Take the head of the queue and re-queue it to the back
+ String nextLdapUrl = ldapUrls.remove();
+ ldapUrls.add(nextLdapUrl);
+
+ DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
+ serverAttempts.set(numAttemptsBeforeFailover);
+ } else {
+ serverAttempts.decrementAndGet();
+ }
+ throw new CommunicationException();
+ }
+ });
+
+ LdapGroupsMapping groupsMapping = getGroupsMapping();
+ groupsMapping.setConf(conf);
+
+ List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
+ assertTrue(groups.isEmpty());
+
+ // Test that we made 6 attempts overall
+ verify(getContext(), times(numAttempts)).search(anyString(),
+ anyString(), any(Object[].class), any(SearchControls.class));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
index e5cd2b6..f51a944 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.security;
-import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@@ -68,7 +67,7 @@ public class TestLdapGroupsMappingWithOneQuery
}
@Test
- public void testGetGroups() throws IOException, NamingException {
+ public void testGetGroups() throws NamingException {
// given a user whose ldap query returns a user object with three "memberOf"
// properties, return an array of strings representing its groups.
String[] testGroups = new String[] {"abc", "xyz", "sss"};
@@ -76,10 +75,9 @@ public class TestLdapGroupsMappingWithOneQuery
}
private void doTestGetGroups(List<String> expectedGroups)
- throws IOException, NamingException {
- Configuration conf = new Configuration();
- // Set this, so we don't throw an exception
- conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+ throws NamingException {
+ String ldapUrl = "ldap://test";
+ Configuration conf = getBaseConf(ldapUrl);
// enable single-query lookup
conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf");
@@ -92,9 +90,7 @@ public class TestLdapGroupsMappingWithOneQuery
Assert.assertEquals(expectedGroups, groups);
// We should have only made one query because single-query lookup is enabled
- verify(getContext(), times(1)).search(anyString(),
- anyString(),
- any(Object[].class),
- any(SearchControls.class));
+ verify(getContext(), times(1)).search(anyString(), anyString(),
+ any(Object[].class), any(SearchControls.class));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9a3aa64/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
index 17d28a5..4a21345 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
@@ -28,7 +28,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@@ -62,7 +61,7 @@ public class TestLdapGroupsMappingWithPosixGroup
}
@Test
- public void testGetGroups() throws IOException, NamingException {
+ public void testGetGroups() throws NamingException {
// The search functionality of the mock context is reused, so we will
// return the user NamingEnumeration first, and then the group
when(getContext().search(anyString(), contains("posix"),
@@ -73,10 +72,9 @@ public class TestLdapGroupsMappingWithPosixGroup
}
private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
- throws IOException, NamingException {
- Configuration conf = new Configuration();
- // Set this, so we don't throw an exception
- conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+ throws NamingException {
+ String ldapUrl = "ldap://test";
+ Configuration conf = getBaseConf(ldapUrl);
conf.set(LdapGroupsMapping.GROUP_SEARCH_FILTER_KEY,
"(objectClass=posixGroup)(cn={0})");
conf.set(LdapGroupsMapping.USER_SEARCH_FILTER_KEY,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: Revert "YARN-8870. [Submarine] Add
submarine installation scripts. (Xun Liu via wangda)"
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
deleted file mode 100644
index 7b3c2a9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description check install user
-## @audience public
-## @stability stable
-function check_install_user()
-{
- if [[ $(id -u) -ne 0 ]];then
- echo "This script must be run with a ROOT user!"
- exit # don't call exit_install()
- fi
-}
-
-## @description exit install
-## @audience public
-## @stability stable
-function exit_install()
-{
- echo "Exit the installation!" | tee -a $LOG
- exit $1
-}
-
-## @description Check if the IP address format is correct
-## @audience public
-## @stability stable
-function valid_ip()
-{
- local ip=$1
- local stat=1
-
- if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
- OIFS=$IFS
- IFS='.'
- ip=($ip)
- IFS=$OIFS
-
- if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then
- stat=$?
- fi
- fi
-
- return $stat
-}
-
-## @description Check if the configuration file configuration is correct
-## @audience public
-## @stability stable
-function check_install_conf()
-{
- echo "Check if the configuration file configuration is correct ..." | tee -a $LOG
-
- # check etcd conf
- hostCount=${#ETCD_HOSTS[@]}
- if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
- echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG
- exit_install
- fi
- for ip in ${ETCD_HOSTS[@]}
- do
- if ! valid_ip $ip; then
- echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
- exit_install
- fi
- done
- echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG
-}
-
-## @description index by EtcdHosts list
-## @audience public
-## @stability stable
-function indexByEtcdHosts() {
- index=0
- while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
- if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
- echo $index
- return
- fi
- let "index++"
- done
- echo ""
-}
-
-## @description get local IP
-## @audience public
-## @stability stable
-function getLocalIP()
-{
- local _ip _myip _line _nl=$'\n'
- while IFS=$': \t' read -a _line ;do
- [ -z "${_line%inet}" ] &&
- _ip=${_line[${#_line[1]}>4?1:2]} &&
- [ "${_ip#127.0.0.1}" ] && _myip=$_ip
- done< <(LANG=C /sbin/ifconfig)
- printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
-}
-
-## @description get ip list
-## @audience public
-## @stability stable
-function get_ip_list()
-{
- array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2)
-
- for ip in ${array[@]}
- do
- LOCAL_HOST_IP_LIST+=(${ip})
- done
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: MAPREDUCE-7165. mapred-site.xml is
misformatted in single node setup document. Contributed by Zhaohui Xin.
Posted by sh...@apache.org.
MAPREDUCE-7165. mapred-site.xml is misformatted in single node setup document. Contributed by Zhaohui Xin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9bfca21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9bfca21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9bfca21
Branch: refs/heads/HDFS-12943
Commit: c9bfca217f4b15a3a367db51147d0dc2075ca274
Parents: 0e36e93
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Nov 30 13:13:23 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Nov 30 13:13:23 2018 +0900
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/SingleCluster.md.vm | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9bfca21/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
index e7a0a08..18fb52d 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
@@ -190,9 +190,6 @@ The following instructions assume that 1. ~ 4. steps of [the above instructions]
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
- </configuration>
-
- <configuration>
<property>
<name>mapreduce.application.classpath</name>
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: Revert "YARN-8870. [Submarine] Add
submarine installation scripts. (Xun Liu via wangda)"
Posted by sh...@apache.org.
Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
This reverts commit 46d6e0016610ced51a76189daeb3ad0e3dbbf94c.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/228156cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/228156cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/228156cf
Branch: refs/heads/HDFS-12943
Commit: 228156cfd1b474988bc4fedfbf7edddc87db41e3
Parents: 9287ab3
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Dec 4 14:13:06 2018 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Dec 4 14:13:06 2018 -0800
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 8 -
.../installation/install.conf | 74 ----
.../installation/install.sh | 116 -----
.../package/calico/calico-node.service | 50 ---
.../installation/package/calico/calicoctl.cfg | 22 -
.../installation/package/docker/daemon.json | 23 -
.../installation/package/docker/docker.service | 35 --
.../installation/package/etcd/etcd.service | 40 --
.../package/hadoop/container-executor.cfg | 41 --
.../installation/package/submarine/submarine.sh | 25 --
.../installation/scripts/calico.sh | 224 ----------
.../installation/scripts/docker.sh | 166 -------
.../installation/scripts/download-server.sh | 42 --
.../installation/scripts/environment.sh | 213 ---------
.../installation/scripts/etcd.sh | 152 -------
.../installation/scripts/hadoop.sh | 117 -----
.../installation/scripts/menu.sh | 444 -------------------
.../installation/scripts/nvidia-docker.sh | 99 -----
.../installation/scripts/nvidia.sh | 120 -----
.../installation/scripts/submarine.sh | 38 --
.../installation/scripts/utils.sh | 123 -----
21 files changed, 2172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 69ab66a..4055acb 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -66,14 +66,6 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation</directory>
- <outputDirectory>/share/hadoop/yarn/submarine-installer</outputDirectory>
- <includes>
- <include>**/*</include>
- </includes>
- <fileMode>0755</fileMode>
- </fileSet>
- <fileSet>
<directory>hadoop-yarn/conf</directory>
<outputDirectory>etc/hadoop</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf
deleted file mode 100644
index 82dcf61..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-##### install config #####
-
-# DNS
-LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf
-YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address
-
-# etcd hosts list
-ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175)
-
-# docker registry ip:port
-DOCKER_REGISTRY="10.120.196.232:5000"
-
-# Start the http download service on the specified server,
-# Will download all the dependencies in the http server,
-# Run the install script on other servers.
-# Automatically download dependencies from http,
-# Solve the problem that all servers are slow to download online.
-# At the same time, you can also manually download the dependencies to the downloads directory.
-# Offline installation of the system
-DOWNLOAD_SERVER_IP="10.120.196.236"
-DOWNLOAD_SERVER_PORT="19000"
-
-# yarn container-executor config
-# How to compile container-executor:
-# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
-# path and enter the command: mvn package -Pnative -DskipTests
-# Only the nodemanager is compiled. The path of the compiled container-executor is:
-# ./target/native/target/usr/local/bin/container-executor
-YARN_CONTAINER_EXECUTOR_PATH="Please enter the full path to the container-executor here"
-
-# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy'
-# in yarn-site.xml, default '/hadoop-yarn'
-YARN_HIERARCHY="/hadoop-yarn"
-
-# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml
-YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local"
-
-# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml
-YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs"
-
-##### System component download url address, Generally do not need to be modified #####
-DOCKER_REPO="https://yum.dockerproject.org/repo/main/centos/7/Packages"
-DOCKER_ENGINE_RPM="docker-engine-1.12.5-1.el7.centos.x86_64.rpm"
-DOCKER_ENGINE_SELINUX_RPM="docker-engine-selinux-1.12.5-1.el7.centos.noarch.rpm"
-
-NVIDIA_DETECT_URL="https://raw.githubusercontent.com/liuxunorg/submarine-installer/packages/nvidia/nvidia-detect"
-NVIDIA_DOCKER_RPM="nvidia-docker-1.0.1-1.x86_64.rpm"
-NVIDIA_DOCKER_RPM_URL="https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/${NVIDIA_DOCKER_RPM}"
-
-ETCD_TAR_GZ="etcd-v3.3.9-linux-amd64.tar.gz"
-ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/v3.3.9/${ETCD_TAR_GZ}"
-
-CALICO_IPV4POOL_CIDR="192.20.0.0"
-CALICO_NETWORK_NAME="calico-network"
-CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico"
-CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam"
-CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh
deleted file mode 100755
index 90b12c2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# description: sumbarine install scripts.
-
-ROOT=$(cd "$(dirname "$0")"; pwd)
-SUBMARINE_INSTALLER_VERSION="v0.7"
-PACKAGE_DIR=${ROOT}/package
-SCRIPTS_DIR=${ROOT}/scripts
-INSTALL_TEMP_DIR=${ROOT}/temp
-DOWNLOAD_DIR=${ROOT}/downloads
-DATE=`date +%Y%m%d-%H:%M:%S`
-LOG=${ROOT}/logs/install.log.`date +%Y%m%d%H%M%S`
-LOCAL_HOST_IP_LIST=()
-LOCAL_HOST_IP=""
-OPERATING_SYSTEM=""
-DOWNLOAD_HTTP=""
-
-# import shell script
-. ${ROOT}/install.conf
-. ${ROOT}/scripts/calico.sh
-. ${ROOT}/scripts/docker.sh
-. ${ROOT}/scripts/download-server.sh
-. ${ROOT}/scripts/environment.sh
-. ${ROOT}/scripts/etcd.sh
-. ${ROOT}/scripts/hadoop.sh
-. ${ROOT}/scripts/menu.sh
-. ${ROOT}/scripts/nvidia.sh
-. ${ROOT}/scripts/nvidia-docker.sh
-. ${ROOT}/scripts/submarine.sh
-. ${ROOT}/scripts/utils.sh
-
-#================================= Main ========================================
-mkdir $ROOT/logs/ -p
-mkdir $INSTALL_TEMP_DIR -p
-mkdir $DOWNLOAD_DIR -p
-
-source /etc/os-release
-OPERATING_SYSTEM=$ID
-
-get_ip_list
-ipCount=${#LOCAL_HOST_IP_LIST[@]}
-if [[ $ipCount -eq 1 ]]; then
- LOCAL_HOST_IP=${LOCAL_HOST_IP_LIST[0]}
-else
- echo -e "Detect the network card IP in the server, \e[31m[${LOCAL_HOST_IP_LIST[@]}]\e[0m"
- echo -n -e "please enter a valid IP address: "
-
- read ipInput
- if ! valid_ip $ipInput; then
- echo -e "you input \e[31m$ipInput\e[0m address format is incorrect! " | tee -a $LOG
- exit_install
- else
- LOCAL_HOST_IP=$ipInput
- fi
-fi
-
-echo -n -e "Please confirm whether the IP address of this machine is \e[31m${LOCAL_HOST_IP}\e[0m?[y|n]"
-read myselect
-if [[ "$myselect" != "y" && "$myselect" != "Y" ]]; then
- exit_install
-fi
-
-check_install_conf
-
-if [[ -n "$DOWNLOAD_SERVER_IP" && -n "$DOWNLOAD_SERVER_PORT" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then
- DOWNLOAD_HTTP="http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}"
-fi
-
-check_install_user
-
-# 清理安装临时目录
-rm $INSTALL_TEMP_DIR/* -rf >>$LOG 2>&1
-
-menu_index="0"
-for ((j=1;;j++))
-do
- menu
- case "$menu_index" in
- "0")
- menu_index="$menu_choice"
- ;;
- "1"|"2"|"3"|"4"|"5")
-# echo "aaaa=$menu_index-$menu_choice"
- menu_process
- if [[ $? = 1 ]]; then
- echo "Press any key to return menu!"
- read
- fi
- ;;
- "a")
- exit_install
- ;;
- "q")
- exit_install
- ;;
- *)
- menu_index="0"
- menu_choice="0"
- menu
- ;;
- esac
-done
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service
deleted file mode 100644
index 744c419..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[Unit]
-Description=calico node
-After=docker.service
-Requires=docker.service
-
-[Service]
-User=root
-PermissionsStartOnly=true
-ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \
- -e ETCD_ENDPOINTS=ETCD_ENDPOINTS_REPLACE \
- -e CALICO_LIBNETWORK_ENABLED=true \
- -e CALICO_NETWORKING_BACKEND=bird \
- -e CALICO_DISABLE_FILE_LOGGING=true \
- -e CALICO_IPV4POOL_CIDR=CALICO_IPV4POOL_CIDR_REPLACE/16 \
- -e CALICO_IPV4POOL_IPIP=always \
- -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \
- -e FELIX_IPV6SUPPORT=false \
- -e FELIX_LOGSEVERITYSCREEN=info \
- -e FELIX_IPINIPMTU=1440 \
- -e FELIX_HEALTHENABLED=true \
- -e IP= \
- -v /etc/calico/ssl:/etc/calico/ssl \
- -v /var/run/calico:/var/run/calico \
- -v /lib/modules:/lib/modules \
- -v /run/docker/plugins:/run/docker/plugins \
- -v /var/run/docker.sock:/var/run/docker.sock \
- -v /var/log/calico:/var/log/calico \
- calico/node:v2.6.2
-ExecStop=/usr/bin/docker rm -f calico-node
-Restart=always
-RestartSec=10
-
-[Install]
-WantedBy=multi-user.target
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg
deleted file mode 100644
index 44ada86..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: calicoApiConfig
-metadata:
-spec:
- datastoreType: "etcdv2"
- etcdEndpoints: ETCD_ENDPOINTS_REPLACE
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json
deleted file mode 100644
index 052fe73..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{
- "insecure-registries": ["DOCKER_REGISTRY_REPLACE"],
- "cluster-store":"CLUSTER_STORE_REPLACE",
- "cluster-advertise":"LOCAL_HOST_IP_REPLACE:2375",
- "dns": ["YARN_DNS_HOST_REPLACE", "LOCAL_DNS_HOST_REPLACE"],
- "hosts": ["tcp://LOCAL_HOST_IP_REPLACE:2375","unix:///var/run/docker.sock"]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service
deleted file mode 100644
index f444f99..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[Unit]
-Description=Docker Application Container Engine
-Documentation=http://docs.docker.io
-
-[Service]
-Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin"
-ExecStart=/usr/bin/dockerd --log-level=error
-ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT
-ExecReload=/bin/kill -s HUP $MAINPID
-Restart=on-failure
-RestartSec=5
-LimitNOFILE=infinity
-LimitNPROC=infinity
-LimitCORE=infinity
-Delegate=yes
-KillMode=process
-
-[Install]
-WantedBy=multi-user.target
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service
deleted file mode 100644
index 5a44ce2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service
+++ /dev/null
@@ -1,40 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[Unit]
-Description=etcd
-Documentation=https://github.com/coreos/etcd
-
-[Service]
-Type=notify
-Restart=always
-RestartSec=5s
-LimitNOFILE=40000
-TimeoutStartSec=0
-
-ExecStart=/usr/bin/etcd \
- --name=ETCD_NODE_NAME_REPLACE \
- --data-dir=/var/lib/etcd \
- --listen-client-urls=http://LOCAL_HOST_REPLACE:2379,http://127.0.0.1:2379 \
- --listen-peer-urls=http://LOCAL_HOST_REPLACE:2380 \
- --advertise-client-urls=http://LOCAL_HOST_REPLACE:2379 \
- --initial-advertise-peer-urls=http://LOCAL_HOST_REPLACE:2380 \
- --initial-cluster=INITIAL_CLUSTER_REPLACE \
- --initial-cluster-token=etcd-token \
- --initial-cluster-state=new
-
-[Install]
-WantedBy=multi-user.target
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg
deleted file mode 100644
index 9d9f0ca..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg
+++ /dev/null
@@ -1,41 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE
-yarn.nodemanager.linux-container-executor.group=yarn
-yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE
-banned.users=root
-allowed.system.users=yarn
-min.user.id=500
-
-[docker]
-module.enabled=true
-docker.binary=/usr/bin/docker
-docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN
-docker.allowed.networks=bridge,host,none,CALICO_NETWORK_NAME_REPLACE
-docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,regex:^nvidia_driver_.*$
-docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE
-docker.privileged-containers.enabled=false
-docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE
-docker.allowed.volume-drivers=nvidia-docker
-docker.allowed.devices=regex:^/dev/nvidia.*$
-
-[gpu]
-module.enabled=true
-
-[cgroups]
-root=/sys/fs/cgroup
-yarn-hierarchy=YARN_HIERARCHY_REPLACE
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh
deleted file mode 100644
index ffbc38f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Solution: Grant user yarn the access to /sys/fs/cgroup/cpu,cpuacct,
-# which is the subfolder of cgroup mount destination.
-chown :yarn -R /sys/fs/cgroup/cpu,cpuacct
-chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct
-
-# If GPUs are used,the access to cgroup devices folder is neede as well
-chown :yarn -R /sys/fs/cgroup/devices
-chmod g+rwx -R /sys/fs/cgroup/devices
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh
deleted file mode 100644
index bd40fcf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh
+++ /dev/null
@@ -1,224 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description download calico bin
-## @audience public
-## @stability stable
-function download_calico_bin()
-{
- # submarin http server
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- MY_CALICOCTL_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calicoctl
- MY_CALICO_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico
- MY_CALICO_IPAM_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico-ipam
- else
- MY_CALICOCTL_DOWNLOAD_URL=${CALICOCTL_DOWNLOAD_URL}
- MY_CALICO_DOWNLOAD_URL=${CALICO_DOWNLOAD_URL}
- MY_CALICO_IPAM_DOWNLOAD_URL=${CALICO_IPAM_DOWNLOAD_URL}
- fi
-
- mkdir -p ${DOWNLOAD_DIR}/calico
-
- if [[ -f ${DOWNLOAD_DIR}/calico/calico ]]; then
- echo "${DOWNLOAD_DIR}/calico/calico is exist."
- else
- echo "download ${MY_CALICO_DOWNLOAD_URL} ..."
- wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_DOWNLOAD_URL}
- fi
-
- if [[ -f ${DOWNLOAD_DIR}/calico/calicoctl ]]; then
- echo "${DOWNLOAD_DIR}/calico is exist."
- else
- echo "download ${MY_CALICOCTL_DOWNLOAD_URL} ..."
- wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICOCTL_DOWNLOAD_URL}
- fi
-
- if [[ -f ${DOWNLOAD_DIR}/calico/calico-ipam ]]; then
- echo "${DOWNLOAD_DIR}/calico/calico-ipam is exist."
- else
- echo "download ${MY_CALICO_IPAM_DOWNLOAD_URL} ..."
- wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_IPAM_DOWNLOAD_URL}
- fi
-}
-
-## @description install calico bin
-## @audience public
-## @stability stable
-function install_calico_bin()
-{
- download_calico_bin
-
- cp -f ${DOWNLOAD_DIR}/calico/calico /usr/bin/calico
- cp -f ${DOWNLOAD_DIR}/calico/calicoctl /usr/bin/calicoctl
- cp -f ${DOWNLOAD_DIR}/calico/calico-ipam /usr/bin/calico-ipam
-
- chmod +x /usr/bin/calico
- chmod +x /usr/bin/calicoctl
- chmod +x /usr/bin/calico-ipam
-}
-
-## @description install calico config
-## @audience public
-## @stability stable
-function install_calico_config()
-{
- mkdir -p /etc/calico
-
- cp -rf ${PACKAGE_DIR}/calico ${INSTALL_TEMP_DIR}/
-
- # 1. replace etcdEndpoints
- # etcdEndpoints: https://10.196.69.173:2379,https://10.196.69.174:2379,https://10.196.69.175:2379
- etcdEndpoints=''
- index=0
- etcdHostsSize=${#ETCD_HOSTS[@]}
- for item in ${ETCD_HOSTS[@]}
- do
- index=$(($index+1))
- etcdEndpoints="${etcdEndpoints}http:\/\/${item}:2379"
- if [[ ${index} -lt ${etcdHostsSize} ]]; then
- etcdEndpoints=${etcdEndpoints}","
- fi
- done
- # echo "etcdEndpoints=${etcdEndpoints}"
- sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calicoctl.cfg >>$LOG
-
- if [[ ! -d /etc/calico ]]; then
- mkdir /etc/calico
- else
- rm -rf /etc/calico/*
- fi
-
- cp -f $INSTALL_TEMP_DIR/calico/calicoctl.cfg /etc/calico/calicoctl.cfg
-
- sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG
- sed -i "s/CALICO_IPV4POOL_CIDR_REPLACE/${CALICO_IPV4POOL_CIDR}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG
- cp $INSTALL_TEMP_DIR/calico/calico-node.service /etc/systemd/system/ >>$LOG
-
- systemctl daemon-reload
- systemctl enable calico-node.service
-}
-
-## @description modify kernel network config
-## @audience public
-## @stability stable
-function kernel_network_config()
-{
- if [ `grep -c "net.ipv4.conf.all.rp_filter=1" /etc/sysctl.conf` -eq '0' ]; then
- echo "net.ipv4.conf.all.rp_filter=1" >>/etc/sysctl.conf
- fi
-
- if [ `grep -c "net.ipv4.ip_forward=1" /etc/sysctl.conf` -eq '0' ]; then
- echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf
- fi
-
- sysctl -p
-}
-
-## @description check if the calico-network exist
-## @audience public
-## @stability stable
-function calico_network_exist()
-{
- local dockerNetwokInfo=`docker network ls --filter NAME=${CALICO_NETWORK_NAME}`
- echo ${dockerNetwokInfo} | grep ${CALICO_NETWORK_NAME}
-}
-
-## @description verification calico
-## @audience public
-## @stability stable
-function verification_calico()
-{
- echo " ===== Check if the network between 2 containers can be connected ====="
- local claicoNetworkExist=`calico_network_exist`
- if [[ "$claicoNetworkExist" = "" ]]; then
- echo "Create a calico network"
- docker network create --driver calico --ipam-driver calico-ipam ${CALICO_NETWORK_NAME}
- else
- echo "calico network ${CALICO_NETWORK_NAME} is exist."
- fi
-
- local verifyA="verify-calico-network-A"
- local verifyAInfo=`containers_exist ${verifyA}`
- if [[ -n "$verifyAInfo" ]]; then
- echo "Delete existing container ${verifyA}."
- docker stop ${verifyA}
- docker rm ${verifyA}
- fi
- echo "Create containers verify-calico-network-A"
- docker run --net ${CALICO_NETWORK_NAME} --name ${verifyA} -tid busybox
-
- local verifyB="verify-calico-network-B"
- local verifyBInfo=`containers_exist ${verifyB}`
- if [[ -n "$verifyBInfo" ]]; then
- echo "Delete existing container ${verifyB}."
- docker stop ${verifyB}
- docker rm ${verifyB}
- fi
- echo "Create containers verify-calico-network-B"
- docker run --net ${CALICO_NETWORK_NAME} --name ${verifyB} -tid busybox
-
- echo -e "\033[33m${verifyA} ping ${verifyB}\033[0m"
- docker exec ${verifyA} ping ${verifyB} -c 5
-}
-
-## @description install calico
-## @audience public
-## @stability stable
-function install_calico()
-{
- kernel_network_config
- install_calico_bin
- install_calico_config
- start_calico
- verification_calico
-}
-
-## @description uninstall calico
-## @audience public
-## @stability stable
-function uninstall_calico()
-{
- echo "stop calico-node.service"
- systemctl stop calico-node.service
-
- echo "rm /usr/bin/calico ..."
- rm /usr/bin/calicoctl
- rm /usr/bin/calico
- rm /usr/bin/calico-ipam
-
- rm -rf /etc/calico/
- rm /etc/systemd/system/calico-node.service
- systemctl daemon-reload
-}
-
-## @description start calico
-## @audience public
-## @stability stable
-function start_calico()
-{
- systemctl restart calico-node.service
- systemctl status calico-node.service
-}
-
-## @description stop calico
-## @audience public
-## @stability stable
-function stop_calico()
-{
- systemctl stop calico-node.service
- systemctl status calico-node.service
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh
deleted file mode 100644
index 26f19cc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description download docker rmp
-## @audience public
-## @stability stable
-function download_docker_rpm()
-{
- # download http server
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- MY_DOCKER_ENGINE_SELINUX_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_SELINUX_RPM}"
- MY_DOCKER_ENGINE_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_RPM}"
- else
- MY_DOCKER_ENGINE_SELINUX_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_SELINUX_RPM}
- MY_DOCKER_ENGINE_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_RPM}
- fi
-
- # download docker rpm
- if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} ]]; then
- echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} is exist."
- else
- echo "download ${MY_DOCKER_ENGINE_SELINUX_RPM} ..."
- wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_SELINUX_RPM}
- fi
-
- if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} ]]; then
- echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} is exist."
- else
- echo "download ${MY_DOCKER_ENGINE_RPM} ..."
- wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_RPM}
- fi
-}
-
-## @description install docker bin
-## @audience public
-## @stability stable
-function install_docker_bin()
-{
- download_docker_rpm
-
- yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM}
- yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM}
-}
-
-## @description uninstall docker bin
-## @audience public
-## @stability stable
-function uninstall_docker_bin()
-{
- download_docker_rpm
-
- yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM}
- yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM}
-}
-
-## @description install docker config
-## @audience public
-## @stability stable
-function install_docker_config()
-{
- rm -rf ${INSTALL_TEMP_DIR}/docker
- cp -rf ${PACKAGE_DIR}/docker ${INSTALL_TEMP_DIR}/
-
- # replace cluster-store
- # "cluster-store":"etcd://10.196.69.173:2379,10.196.69.174:2379,10.196.69.175:2379"
- # char '/' need to escape '\/'
- clusterStore="etcd:\/\/"
- index=1
- etcdHostsSize=${#ETCD_HOSTS[@]}
- for item in ${ETCD_HOSTS[@]}
- do
- clusterStore="${clusterStore}${item}:2379"
- if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then
- clusterStore=${clusterStore}","
- fi
- index=$(($index+1))
- done
- echo "clusterStore=${clusterStore}"
- sed -i "s/CLUSTER_STORE_REPLACE/${clusterStore}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
-
- sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
- sed -i "s/LOCAL_HOST_IP_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
- sed -i "s/YARN_DNS_HOST_REPLACE/${YARN_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
- sed -i "s/LOCAL_DNS_HOST_REPLACE/${LOCAL_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
-
- # Delete the ASF license comment in the daemon.json file, otherwise it will cause a json format error.
- sed -i '1,16d' $INSTALL_TEMP_DIR/docker/daemon.json
-
- if [ ! -d "/etc/docker" ]; then
- mkdir /etc/docker
- fi
-
- cp $INSTALL_TEMP_DIR/docker/daemon.json /etc/docker/
- cp $INSTALL_TEMP_DIR/docker/docker.service /etc/systemd/system/ >>$LOG
-}
-
-## @description install docker
-## @audience public
-## @stability stable
-function install_docker()
-{
- install_docker_bin
- install_docker_config
-
- systemctl daemon-reload
- systemctl enable docker.service
-}
-
-## @description unstall docker
-## @audience public
-## @stability stable
-function uninstall_docker()
-{
- echo "stop docker service"
- systemctl stop docker
-
- echo "remove docker"
- uninstall_docker_bin
-
- rm /etc/docker/daemon.json >>$LOG
- rm /etc/systemd/system/docker.service >>$LOG
-
- systemctl daemon-reload
-}
-
-## @description start docker
-## @audience public
-## @stability stable
-function start_docker()
-{
- systemctl restart docker
- systemctl status docker
- docker info
-}
-
-## @description stop docker
-## @audience public
-## @stability stable
-function stop_docker()
-{
- systemctl stop docker
- systemctl status docker
-}
-
-## @description check if the containers exist
-## @audience public
-## @stability stable
-function containers_exist()
-{
- local dockerContainersInfo=`docker ps -a --filter NAME=$1`
- echo ${dockerContainersInfo} | grep $1
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh
deleted file mode 100644
index 62af827..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description start download server
-## @audience public
-## @stability stable
-function start_download_server()
-{
- if [[ "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then
- echo -e "\033[31mERROR: Only $DOWNLOAD_SERVER_IP can start the download service.\033[0m"
- return 1
- fi
-
- echo -e "You can put the install package file in the \033[34m${DOWNLOAD_DIR}\033[0m folder first, Or automatic download."
- echo -n "Do you want to start download http server?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- download_etcd_bin
- download_calico_bin
- download_docker_rpm
- download_nvidia_driver
- download_nvidia_docker_bin
- download_yarn_container_executor
-
- python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT}
- fi
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh
deleted file mode 100644
index 17f59b2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh
+++ /dev/null
@@ -1,213 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description check operation System
-## @audience public
-## @stability stable
-function check_operationSystem()
-{
- echo -e "The submarine assembly support \033[32m[centos-release-7-3.1611.el7.centos.x86_64]\033[0m or higher operating system version."
-
- case ${OPERATING_SYSTEM} in
- centos)
- local operationSystemVersion=`rpm --query centos-release`
- echo -e "The current operating system version is \e[31m[${operationSystemVersion}]\e[0m" | tee -a $LOG
- ;;
- *)
- echo -e "\033[31mWARN: The submarine assembly Unsupported [${OPERATING_SYSTEM}] operating system\033[0m"
- ;;
- esac
-}
-
-## @description update operation System Kernel
-## @audience public
-## @stability stable
-function update_operationSystemKernel()
-{
- echo "If the server is unable to connect to the network, execute the following command yourself:
- wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm
- rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm"
-
- echo -n "Do you want to kernel upgrades?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- echo "Now try to use the yum command for kernel upgrades ..."
- yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r)
-
- local kernelVersion=`uname -r`
- echo -e "After the upgrade, the operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG
- fi
-}
-
-## @description check operation system kernel
-## @audience public
-## @stability stable
-function check_operationSystemKernel()
-{
-case ${OPERATING_SYSTEM} in
-centos)
- local kernelVersion=`uname -r`
-
- echo -e "Submarine support operating system kernel version is \033[32m 3.10.0-514.el7.x86_64 \033[0m" | tee -a $LOG
- echo -e "Current operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG
-
- update_operationSystemKernel
- ;;
-*)
- echo -e "\033[31m WARN: The submarine assembly Unsupported operating system [${OPERATING_SYSTEM}] \033[0m"
- ;;
-esac
-}
-
-## @description get gcc version
-## @audience public
-## @stability stable
-function get_gcc_version()
-{
- local gccVersion=`gcc --version`
- version=${gccVersion%Copyright*}
- echo $version
-}
-
-## @description install gcc
-## @audience public
-## @stability stable
-function install_gcc()
-{
- echo -n "Do you want to install gcc?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]; then
- echo "Execute the yum install gcc make g++ command"
- yum install gcc make g++
-
- local gccVersion=`gcc --version`
- echo -e "After the install, the gcc version is \e[31m${gccVersion}\e[0m" | tee -a $LOG
- fi
-}
-
-## @description check gcc Version
-## @audience public
-## @stability stable
-function check_gccVersion()
-{
- local gccVersionInfo=`gcc --version`
- local gccVersion=${gccVersionInfo%Copyright*}
-
- if [[ "$gccVersion" = "" ]]; then
- echo "The gcc was not installed on the system. Automated installation ..."
- install_gcc
- else
- echo -e "Submarine gcc version need \033[34mgcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\033[0m or higher."
- echo -e "Current gcc version was \033[34m${gccVersion}\033[0m"
- fi
-}
-
-## @description check GPU
-## @audience public
-## @stability stable
-function check_GPU()
-{
- gpuInfo=`lspci | grep -i nvidia`
-
- if [[ "$gpuInfo" = "" ]]; then
- echo -e "\033[31mERROR: The system did not detect the GPU graphics card.\033[0m"
- else
- echo -e "\033[32mINFO: The system detect the GPU graphics card.\033[0m"
- fi
-}
-
-## @description check user group
-## @audience public
-## @stability stable
-function check_userGroup()
-{
- echo -e "check hadoop user group ..."
-
- echo -e "Hadoop runs the required user [hdfs, mapred, yarn] and groups [hdfs, mapred, yarn, hadoop] installed by ambari."
- echo -e "If you are not using ambari for hadoop installation,
-then you can add the user and group by root by executing the following command:
-\033[34madduser hdfs
-adduser mapred
-adduser yarn
-addgroup hadoop
-usermod -aG hdfs,hadoop hdfs
-usermod -aG mapred,hadoop mapred
-usermod -aG yarn,hadoop yarn
-usermod -aG hdfs,hadoop hadoop
-groupadd docker
-usermod -aG docker yarn
-usermod -aG docker hadoop\033[0m\n"
-
- echo -e "check docker user group ..."
- # check user group
- DOCKER_USER_GROUP='docker'
- egrep "^${DOCKER_USER_GROUP}" /etc/group >& /dev/null
- if [[ $? -ne 0 ]]; then
- echo -e "user group ${DOCKER_USER_GROUP} does not exist, Please execute the following command:"
- echo -e "\033[34mgroupadd $DOCKER_USER_GROUP\033[0m"
- fi
-
- # check user
- USER_GROUP=(yarn hadoop)
- for user in ${USER_GROUP[@]}
- do
- egrep "^${user}" /etc/passwd >& /dev/null
- if [[ $? -ne 0 ]]; then
- echo -e "User ${user} does not exist, Please execute the following command:"
- echo -e "\033[34madduser ${user}\033[0m"
- echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m"
- fi
-
- echo -e "Please execute the following command:"
- echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m"
- done
-}
-
-## @description Some preparatory work for nvidia driver installation
-## @audience public
-## @stability stable
-function prepare_nvidia_environment()
-{
- echo "prepare nvidia environment ..."
-
- yum -y update
- yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r)
-
- yum -y install epel-release
- yum -y install dkms
-
- echo -e "\033[34m ===== Please manually execute the following command =====
-# 1. Disable nouveau
-# Add the content 'rd.driver.blacklist=nouveau nouveau.modeset=0'
-# to the 'GRUB_CMDLINE_LINUX' configuration item in the /etc/default/grub file.
-root:> vi /etc/default/grub
-vi:> GRUB_CMDLINE_LINUX=\"rd.driver.blacklist=nouveau nouveau.modeset=0 ...\"
-
-# 2. Generate configuration
-root:> grub2-mkconfig -o /boot/grub2/grub.cfg
-
-# 3. Open (new) /etc/modprobe.d/blacklist.conf, add content 'blacklist nouveau'
-root:> vi /etc/modprobe.d/blacklist.conf
-vi:> blacklist nouveau
-
-# 4. Update configuration and reboot
-root:> mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r)-nouveau.img
-root:> dracut /boot/initramfs-$(uname -r).img $(uname -r)
-root:> reboot
-\033[0m"
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh
deleted file mode 100644
index 48df80e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description download etcd bin
-## @audience public
-## @stability stable
-function download_etcd_bin()
-{
- # my download http server
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- MY_ETCD_DOWNLOAD_URL="${DOWNLOAD_HTTP}/downloads/etcd/${ETCD_TAR_GZ}"
- else
- MY_ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL}
- fi
-
- if [[ -f "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" ]]; then
- echo "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} is exist."
- else
- echo "download ${MY_ETCD_DOWNLOAD_URL} ..."
- wget -P ${DOWNLOAD_DIR}/etcd ${MY_ETCD_DOWNLOAD_URL}
- fi
-}
-
-## @description install etcd bin
-## @audience public
-## @stability stable
-function install_etcd_bin()
-{
- download_etcd_bin
-
- # install etcd bin
- mkdir -p ${INSTALL_TEMP_DIR}
- rm -rf ${INSTALL_TEMP_DIR}/etcd-*-linux-amd6
- tar zxvf ${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} -C ${INSTALL_TEMP_DIR}
-
- cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcd /usr/bin
- cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcdctl /usr/bin
-
- mkdir -p /var/lib/etcd
- chmod -R a+rw /var/lib/etcd
-}
-
-## @description install etcd config
-## @audience public
-## @stability stable
-function install_etcd_config()
-{
- # config etcd.service
- rm -rf ${INSTALL_TEMP_DIR}/etcd
- cp -rf ${PACKAGE_DIR}/etcd ${INSTALL_TEMP_DIR}/
-
- # 1. Replace name with ETCD_NODE_NAME_REPLACE based on the location of the local IP in $ETCD_HOSTS
- indexEtcdList=$(indexByEtcdHosts ${LOCAL_HOST_IP})
- # echo ${indexEtcdList}
- etcdNodeName="etcdnode${indexEtcdList}"
- # echo ${etcdNodeName}
- sed -i "s/ETCD_NODE_NAME_REPLACE/${etcdNodeName}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG
-
- # 2. Replace local IP address
- sed -i "s/LOCAL_HOST_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG
-
- # 3. Replace the initial-cluster parameter
- # --initial-cluster=etcdnode1=http://10.196.69.173:2380,etcdnode2=http://10.196.69.174:2380,etcdnode3=http://10.196.69.175:2380 \
- initialCluster=''
- index=0
- etcdHostsSize=${#ETCD_HOSTS[@]}
- for item in ${ETCD_HOSTS[@]}
- do
- # char '/' need to escape '\/'
- initialCluster="${initialCluster}etcdnode${index}=http:\/\/${item}:2380"
- if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then
- initialCluster=${initialCluster}","
- fi
- index=$(($index+1))
- done
- #echo "initialCluster=${initialCluster}"
- sed -i "s/INITIAL_CLUSTER_REPLACE/${initialCluster}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG
-
- cp $INSTALL_TEMP_DIR/etcd/etcd.service /etc/systemd/system/ >>$LOG
-}
-
-## @description install etcd
-## @audience public
-## @stability stable
-function install_etcd()
-{
- index=$(indexByEtcdHosts ${LOCAL_HOST_IP})
- if [ -z "$index" ]; then
- echo -e "STOP: This host\033[31m[${LOCAL_HOST_IP}]\033[0m is not in the ETCD server list\033[31m[${ETCD_HOSTS[@]}]\033[0m"
- return 1
- fi
-
- install_etcd_bin
-
- install_etcd_config
-
- systemctl daemon-reload
- systemctl enable etcd.service
-}
-
-## @description uninstall etcd
-## @audience public
-## @stability stable
-function uninstall_etcd()
-{
- echo "stop etcd.service"
- systemctl stop etcd.service
-
- echo "rm etcd ..."
- rm /usr/bin/etcd
- rm /usr/bin/etcdctl
- rm -rf /var/lib/etcd
- rm /etc/systemd/system/etcd.service
-
- systemctl daemon-reload
-}
-
-## @description start etcd
-## @audience public
-## @stability stable
-function start_etcd()
-{
- systemctl restart etcd.service
-
- echo " ===== Check the status of the etcd service ====="
- echo " exec etcdctl cluster-health"
- etcdctl cluster-health
- echo " exec etcdctl cluster-health"
- etcdctl member list
-}
-
-## @description stop etcd
-## @audience public
-## @stability stable
-function stop_etcd()
-{
- systemctl stop etcd.service
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh
deleted file mode 100644
index 9df5d7e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description install yarn
-## @audience public
-## @stability stable
-function install_yarn()
-{
- install_yarn_container_executor
- install_yarn_config
-}
-
-## @description uninstall yarn
-## @audience public
-## @stability stable
-function uninstall_yarn()
-{
- rm -rf /etc/yarn/sbin/Linux-amd64-64/*
- rm -rf /etc/yarn/sbin/etc/hadoop/*
-}
-
-## @description download yarn container executor
-## @audience public
-## @stability stable
-function download_yarn_container_executor()
-{
- # my download http server
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- MY_YARN_CONTAINER_EXECUTOR_PATH="${DOWNLOAD_HTTP}/downloads/hadoop/container-executor"
- else
- MY_YARN_CONTAINER_EXECUTOR_PATH=${YARN_CONTAINER_EXECUTOR_PATH}
- fi
-
- if [ ! -d "${DOWNLOAD_DIR}/hadoop" ]; then
- mkdir -p ${DOWNLOAD_DIR}/hadoop
- fi
-
- if [[ -f "${DOWNLOAD_DIR}/hadoop/container-executor" ]]; then
- echo "${DOWNLOAD_DIR}/hadoop/container-executor is exist."
- else
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- echo "download ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..."
- wget -P ${DOWNLOAD_DIR}/hadoop ${MY_YARN_CONTAINER_EXECUTOR_PATH}
- else
- echo "copy ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..."
- cp ${MY_YARN_CONTAINER_EXECUTOR_PATH} ${DOWNLOAD_DIR}/hadoop/
- fi
- fi
-}
-
-## @description install yarn container executor
-## @audience public
-## @stability stable
-function install_yarn_container_executor()
-{
- echo "install yarn container executor file ..."
-
- download_yarn_container_executor
-
- if [ ! -d "/etc/yarn/sbin/Linux-amd64-64" ]; then
- mkdir -p /etc/yarn/sbin/Linux-amd64-64
- fi
- if [ -f "/etc/yarn/sbin/Linux-amd64-64/container-executor" ]; then
- rm /etc/yarn/sbin/Linux-amd64-64/container-executor
- fi
-
- cp -f ${DOWNLOAD_DIR}/hadoop/container-executor /etc/yarn/sbin/Linux-amd64-64
-
- sudo chmod 6755 /etc/yarn/sbin/Linux-amd64-64
- sudo chown :yarn /etc/yarn/sbin/Linux-amd64-64/container-executor
- sudo chmod 6050 /etc/yarn/sbin/Linux-amd64-64/container-executor
-}
-
-## @description install yarn config
-## @audience public
-## @stability stable
-function install_yarn_config()
-{
- echo "install yarn config file ..."
-
- cp -R ${PACKAGE_DIR}/hadoop ${INSTALL_TEMP_DIR}/
-
- find="/"
- replace="\/"
- escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace}
- escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace}
- escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace}
-
- sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
- sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
- sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
- sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
- sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
-
- # Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error.
- sed -i '1,16d' $INSTALL_TEMP_DIR/hadoop/container-executor.cfg
-
- if [ ! -d "/etc/yarn/sbin/etc/hadoop" ]; then
- mkdir -p /etc/yarn/sbin/etc/hadoop
- fi
-
- cp -f $INSTALL_TEMP_DIR/hadoop/container-executor.cfg /etc/yarn/sbin/etc/hadoop/
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh
deleted file mode 100644
index 68f91c1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh
+++ /dev/null
@@ -1,444 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description main menu
-## @audience public
-## @stability stable
-main_menu()
-{
-cat<<MENULIST
-====================================================================================
- SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
-
-HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
-====================================================================================
-[Main menu]
-------------------------------------------------------------------------------------
-MENULIST
-echo -e " \e[32m1.prepare system environment [..]\e[0m"
-echo -e " \e[32m2.install component [..]\e[0m"
-echo -e " \e[32m3.uninstall component [..]\e[0m"
-echo -e " \e[32m4.start component [..]\e[0m"
-echo -e " \e[32m5.stop component [..]\e[0m"
-echo -e " \e[32m6.start download server [..]\e[0m"
-echo -e ""
-echo -e " \e[32mq.quit\e[0m"
-cat<<MENULIST
-====================================================================================
-MENULIST
-
-echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m6\e[0m,\e[32mq\e[0m(quit)]:"
-}
-
-## @description check menu
-## @audience public
-## @stability stable
-check_menu()
-{
-cat<<MENULIST
-====================================================================================
- SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
-
-HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
-====================================================================================
-[Main menu] > [prepare system environment]
-------------------------------------------------------------------------------------
-MENULIST
-echo -e " \e[32m1.prepare operation system\e[0m"
-echo -e " \e[32m2.prepare operation system kernel\e[0m"
-echo -e " \e[32m3.prepare GCC version\e[0m"
-echo -e " \e[32m4.check GPU\e[0m"
-echo -e " \e[32m5.prepare user&group\e[0m"
-echo -e " \e[32m6.prepare nvidia environment\e[0m"
-echo -e ""
-echo -e " \e[32mb.back main menu\e[0m"
-cat<<MENULIST
-====================================================================================
-MENULIST
-
-echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m5\e[0m,\e[32mb\e[0m(back)]:"
-}
-
-## @description install menu
-## @audience public
-## @stability stable
-install_menu()
-{
-cat<<MENULIST
-====================================================================================
- SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
-
-HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
-====================================================================================
-[Main menu] > [install component]
-------------------------------------------------------------------------------------
-MENULIST
-echo -e " \e[32m1.instll etcd\e[0m"
-echo -e " \e[32m2.instll docker\e[0m"
-echo -e " \e[32m3.instll calico network\e[0m"
-echo -e " \e[32m4.instll nvidia driver\e[0m"
-echo -e " \e[32m5.instll nvidia docker\e[0m"
-echo -e " \e[32m6.instll yarn container-executor\e[0m"
-echo -e " \e[32m7.instll submarine autorun script\e[0m"
-echo -e ""
-echo -e " \e[32mb.back main menu\e[0m"
-cat<<MENULIST
-====================================================================================
-MENULIST
-
-echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:"
-}
-
-## @description unstall menu
-## @audience public
-## @stability stable
-uninstall_menu()
-{
-cat<<MENULIST
-====================================================================================
- SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
-
-HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
-====================================================================================
-[Main menu] > [uninstll component]
-------------------------------------------------------------------------------------
-MENULIST
-echo -e " \e[32m1.uninstll etcd\e[0m"
-echo -e " \e[32m2.uninstll docker\e[0m"
-echo -e " \e[32m3.uninstll calico network\e[0m"
-echo -e " \e[32m4.uninstll nvidia driver\e[0m"
-echo -e " \e[32m5.uninstll nvidia docker\e[0m"
-echo -e " \e[32m6.uninstll yarn container-executor\e[0m"
-echo -e " \e[32m7.uninstll submarine autorun script\e[0m"
-echo -e ""
-echo -e " \e[32mb.back main menu\e[0m"
-cat<<MENULIST
-====================================================================================
-MENULIST
-
-echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:"
-}
-
-## @description start menu
-## @audience public
-## @stability stable
-start_menu()
-{
-cat<<MENULIST
-====================================================================================
- SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
-
-HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
-====================================================================================
-[Main menu] > [stop component]
-------------------------------------------------------------------------------------
-MENULIST
-echo -e " \e[32m1.start etcd\e[0m"
-echo -e " \e[32m2.start docker\e[0m"
-echo -e " \e[32m3.start calico network\e[0m"
-echo -e ""
-echo -e " \e[32mb.back main menu\e[0m"
-cat<<MENULIST
-====================================================================================
-MENULIST
-
-echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:"
-}
-
-## @description stop menu
-## @audience public
-## @stability stable
-stop_menu()
-{
-cat<<MENULIST
-====================================================================================
- SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
-
-HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
-====================================================================================
-[Main menu] > [stop component]
-------------------------------------------------------------------------------------
-MENULIST
-echo -e " \e[32m1.stop etcd\e[0m"
-echo -e " \e[32m2.stop docker\e[0m"
-echo -e " \e[32m3.stop calico network\e[0m"
-echo -e ""
-echo -e " \e[32mb.back main menu\e[0m"
-cat<<MENULIST
-====================================================================================
-MENULIST
-
-echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:"
-}
-
-## @description menu operation
-## @audience public
-## @stability stable
-menu_index="0"
-menu()
-{
- clear
- # echo "menu_index-menu_choice=$menu_index-$menu_choice"
- case $menu_index in
- "0")
- main_menu
- ;;
- "1")
- check_menu
- ;;
- "2")
- install_menu
- ;;
- "3")
- uninstall_menu
- ;;
- "4")
- start_menu
- ;;
- "5")
- stop_menu
- ;;
- "6")
- start_download_server
- ;;
- "q")
- exit 1
- ;;
- *)
- echo "error input!"
- menu_index="0"
- menu_choice="0"
- main_menu
- ;;
- esac
-
- read menu_choice
-}
-
-## @description menu process
-## @audience public
-## @stability stable
-menu_process()
-{
- process=0
- unset myselect
- # echo "debug=$menu_index-$menu_choice"
- case "$menu_index-$menu_choice" in
- "1-b"|"2-b"|"3-b"|"4-b"|"5-b"|"6-b")
- menu_index="0"
- menu_choice="0"
- ;;
-# check system environment
- "1-1")
- myselect="y"
- check_operationSystem
- ;;
- "1-2")
- myselect="y"
- check_operationSystemKernel
- ;;
- "1-3")
- myselect="y"
- check_gccVersion
- ;;
- "1-4")
- myselect="y"
- check_GPU
- ;;
- "1-5")
- myselect="y"
- check_userGroup
- ;;
- "1-6")
- myselect="y"
- prepare_nvidia_environment
- ;;
-# install component
- "2-1")
- echo -n "Do you want to install etcd?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- install_etcd
- fi
- ;;
- "2-2")
- echo -n "Do you want to install docker?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- install_docker
- fi
- ;;
- "2-3")
- echo -n "Do you want to install calico network?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- install_calico
- fi
- ;;
- "2-4")
- echo -n "Do you want to install nvidia driver?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- install_nvidia
- fi
- ;;
- "2-5")
- echo -n "Do you want to install nvidia docker?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- install_nvidia_docker
- fi
- ;;
- "2-6")
- echo -n "Do you want to install yarn container-executor?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- install_yarn
- fi
- ;;
- "2-7")
- echo -n "Do you want to install submarine auto start script?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- install_submarine
- fi
- ;;
-# uninstall component
- "3-1")
- echo -n "Do you want to uninstall etcd?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- uninstall_etcd
- fi
- ;;
- "3-2")
- echo -n "Do you want to uninstall docker?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- uninstall_docker
- fi
- ;;
- "3-3")
- echo -n "Do you want to uninstall calico network?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- uninstall_calico
- fi
- ;;
- "3-4")
- echo -n "Do you want to uninstall nvidia driver?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- uninstall_nvidia
- fi
- ;;
- "3-5")
- echo -n "Do you want to uninstall nvidia docker?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- uninstall_nvidia_docker
- fi
- ;;
- "3-6")
- echo -n "Do you want to uninstall yarn container-executor?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- uninstall_yarn
- fi
- ;;
- "3-7")
- echo -n "Do you want to uninstall submarine autostart script?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- uninstall_submarine
- fi
- ;;
-# startup component
- "4-1")
- echo -n "Do you want to startup etcd?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- start_etcd
- fi
- ;;
- "4-2")
- echo -n "Do you want to startup docker?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- start_docker
- fi
- ;;
- "4-3")
- echo -n "Do you want to startup calico network?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- start_calico
- fi
- ;;
-# stop component
- "5-1")
- echo -n "Do you want to stop etcd?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- stop_etcd
- fi
- ;;
- "5-2")
- echo -n "Do you want to stop docker?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- stop_docker
- fi
- ;;
- "5-3")
- echo -n "Do you want to stop calico network?[y|n]"
- read myselect
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- stop_calico
- fi
- ;;
- esac
-
- if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
- then
- process=1
- fi
-
-# echo "process=$process"
- return $process
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh
deleted file mode 100644
index 3441f28..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description download nvidia docker bin
-## @audience public
-## @stability stable
-function download_nvidia_docker_bin()
-{
- # download http server
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- MY_NVIDIA_DOCKER_RPM_URL="${DOWNLOAD_HTTP}/downloads/nvidia-docker/${NVIDIA_DOCKER_RPM}"
- else
- MY_NVIDIA_DOCKER_RPM_URL=${NVIDIA_DOCKER_RPM_URL}
- fi
-
- if [[ -f "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM}" ]]; then
- echo "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM} is exist."
- else
- echo "download ${MY_NVIDIA_DOCKER_RPM_URL} ..."
- wget -P ${DOWNLOAD_DIR}/nvidia-docker/ ${MY_NVIDIA_DOCKER_RPM_URL}
- fi
-}
-
-## @description install nvidia docker
-## @audience public
-## @stability stable
-function install_nvidia_docker()
-{
- download_nvidia_docker_bin
-
- sudo rpm -i ${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM}
-
- echo -e "\033[32m===== Start nvidia-docker =====\033[0m"
- sudo systemctl start nvidia-docker
-
- echo -e "\033[32m===== Check nvidia-docker status =====\033[0m"
- systemctl status nvidia-docker
-
- echo -e "\033[32m===== Check nvidia-docker log =====\033[0m"
- journalctl -u nvidia-docker
-
- echo -e "\033[32m===== Test nvidia-docker-plugin =====\033[0m"
- curl http://localhost:3476/v1.0/docker/cli
-
- # create nvidia driver library path
- if [ ! -d "/var/lib/nvidia-docker/volumes/nvidia_driver" ]; then
- echo "WARN: /var/lib/nvidia-docker/volumes/nvidia_driver folder path is not exist!"
- mkdir -p /var/lib/nvidia-docker/volumes/nvidia_driver
- fi
-
- local nvidiaVersion=`get_nvidia_version`
- echo -e "\033[31m nvidia detect version is ${nvidiaVersion}\033[0m"
-
- mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}
- mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin
- mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64
-
- cp /usr/bin/nvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin
- cp /usr/lib64/libcuda* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64
- cp /usr/lib64/libnvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64
-
- echo -e "\033[32m===== Please manually execute the following command =====\033[0m"
- echo -e "\033[32mshell:> nvidia-docker run --rm ${DOCKER_REGISTRY}/nvidia/cuda:9.0-devel nvidia-smi
-# If you don't see the list of graphics cards above, the NVIDIA driver installation failed. =====
-\033[0m"
-
- echo -e "\033[32m===== Please manually execute the following command =====\033[0m"
- echo -e "\033[32m# Test with tf.test.is_gpu_available()
-shell:> nvidia-docker run -it ${DOCKER_REGISTRY}/tensorflow/tensorflow:1.9.0-gpu bash
-# In docker container
-container:> python
-python:> import tensorflow as tf
-python:> tf.test.is_gpu_available()
-python:> exit()
-\033[0m"
-}
-
-## @description uninstall nvidia docker
-## @audience public
-## @stability stable
-function uninstall_nvidia_docker()
-{
- echo "This method is not implemented."
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh
deleted file mode 100644
index 7b2b1de..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh
+++ /dev/null
@@ -1,120 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-nvidia_run_file=""
-
-## @description get nvidia version
-## @audience public
-## @stability stable
-function get_nvidia_version()
-{
- chmod +x ${DOWNLOAD_DIR}/nvidia/nvidia-detect
- local nvidia_detect_info=`${DOWNLOAD_DIR}/nvidia/nvidia-detect -v`
- echo $nvidia_detect_info | sed "s/^.*This device requires the current \([0-9.]*\).*/\1/"
-}
-
-## @description download nvidia driver
-## @audience public
-## @stability stable
-function download_nvidia_driver()
-{
- # submarin http server
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- MY_NVIDIA_DETECT_URL="${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect"
- else
- MY_NVIDIA_DETECT_URL=${NVIDIA_DETECT_URL}
- fi
-
- # download nvidia_detect
- if [[ -f "${DOWNLOAD_DIR}/nvidia/nvidia-detect" ]]; then
- echo "${DOWNLOAD_DIR}/nvidia/nvidia-detect is exist."
- else
- echo "download ${MY_NVIDIA_DETECT_URL} ..."
- wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DETECT_URL}
- fi
-
- echo "execution nvidia-detect to check the graphics card ..."
- local nvidiaVersion=`get_nvidia_version`
- echo -e "detect nvidia version is \033[31m${nvidiaVersion}\033[0m"
-
- # download NVIDIA driver
- if [[ "$nvidiaVersion" = "" ]]; then
- echo -e "\033[31mERROR: No graphics card device detected.\033[0m"
- return 1
- else
- nvidia_run_file="NVIDIA-Linux-x86_64-${nvidiaVersion}.run"
-
- # submarin http server
- if [[ -n "$DOWNLOAD_HTTP" ]]; then
- MY_NVIDIA_DRIVER_RUN_URL="${DOWNLOAD_HTTP}/downloads/nvidia/${nvidia_run_file}"
- else
- # http://us.download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run
- MY_NVIDIA_DRIVER_RUN_URL="http://us.download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/${nvidia_run_file}"
- fi
-
- if [[ -f ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} ]]; then
- echo "NVIDIA driver files already exist in the ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} directory."
- echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia/${nvidia_run_file} file is complete and can be used normally. ====="
- else
- echo "Download the NVIDIA driver from the ${MY_NVIDIA_DRIVER_RUN_URL}"
- wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DRIVER_RUN_URL}
- fi
- fi
-}
-
-## @description install nvidia
-## @audience public
-## @stability stable
-function install_nvidia()
-{
- download_nvidia_driver
-
- # Confirm that the system disables nouveau
- local disable_nouveau_info=`lsmod | grep nouveau`
- if [[ "$disable_nouveau_info" = "" ]]; then
- echo "===== Start installing the NVIDIA driver ====="
- echo -e "Some options during the installation
-Would you like to register the kernel module sources with DKMS?
- This will allow DKMS to automatically build a new module, if you install a different kernel later. \033[33m[Yes]\033[0m
-Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m
-centos Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m
-Would you like to run the nvidia-xconfig utility to automatically update your X configuration file... \033[33m[No]\033[0m"
- sleep 2
- sh ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file}
- else
- echo -e "ERROR: Nouveau is not disabled"
- return 1
- fi
-
- echo -e "\033[32m===== execute nvidia-smi. You should be able to see the list of graphics cards =====\033[0m"
- sleep 1
- nvidia-smi
-}
-
-## @description uninstall nvidia
-## @audience public
-## @stability stable
-function uninstall_nvidia()
-{
- if [ ! -f "/usr/bin/nvidia-uninstall" ]; then
- echo -e "\033[31mERROR: /usr/bin/nvidia-uninstall file is not exist!\033[0m"
- return 1
- fi
-
- echo -e "execute /usr/bin/nvidia-uninstall"
- /usr/bin/nvidia-uninstall
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh
deleted file mode 100644
index b73d5e7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description install submarine autorun script
-## @audience public
-## @stability stable
-function install_submarine()
-{
- cp ${PACKAGE_DIR}/submarine/submarine.sh /etc/rc.d/init.d/submarine.sh
- chmod +x /etc/rc.d/init.d/submarine.sh
- chmod +x /etc/rc.d/rc.local
-
- if [ `grep -c "/etc/rc.d/init.d/submarine.sh" /etc/rc.d/rc.local` -eq '0' ]; then
- echo "/etc/rc.d/init.d/submarine.sh">> /etc/rc.d/rc.local
- fi
-}
-
-## @description uninstall submarine autorun script
-## @audience public
-## @stability stable
-function uninstall_submarine()
-{
- rm /etc/rc.d/init.d/submarine.sh
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDFS-14119. Improve GreedyPlanner
Parameter Logging. Contributed by Beluga Behr.
Posted by sh...@apache.org.
HDFS-14119. Improve GreedyPlanner Parameter Logging. Contributed by Beluga Behr.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69489ff2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69489ff2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69489ff2
Branch: refs/heads/HDFS-12943
Commit: 69489ff2d18a26be9c56d632672079e134f1cd99
Parents: c9a3aa6
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Mon Dec 3 12:55:52 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Mon Dec 3 12:55:52 2018 -0800
----------------------------------------------------------------------
.../diskbalancer/planner/GreedyPlanner.java | 24 ++++++++------------
1 file changed, 10 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69489ff2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
index 568c1e6..3f97345 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
@@ -64,7 +64,7 @@ public class GreedyPlanner implements Planner {
*/
@Override
public NodePlan plan(DiskBalancerDataNode node) throws Exception {
- long startTime = Time.monotonicNow();
+ final long startTime = Time.monotonicNow();
NodePlan plan = new NodePlan(node.getDataNodeName(),
node.getDataNodePort());
LOG.info("Starting plan for Node : {}:{}",
@@ -75,12 +75,10 @@ public class GreedyPlanner implements Planner {
}
}
- long endTime = Time.monotonicNow();
- String message = String
- .format("Compute Plan for Node : %s:%d took %d ms ",
- node.getDataNodeName(), node.getDataNodePort(),
- endTime - startTime);
- LOG.info(message);
+ final long endTime = Time.monotonicNow();
+ LOG.info("Compute Plan for Node : {}:{} took {} ms",
+ node.getDataNodeName(), node.getDataNodePort(), endTime - startTime);
+
return plan;
}
@@ -117,21 +115,19 @@ public class GreedyPlanner implements Planner {
applyStep(nextStep, currentSet, lowVolume, highVolume);
if (nextStep != null) {
- LOG.debug("Step : {} ", nextStep.toString());
+ LOG.debug("Step : {} ", nextStep);
plan.addStep(nextStep);
}
}
- String message = String
- .format("Disk Volume set %s Type : %s plan completed.",
- currentSet.getSetID(),
- currentSet.getVolumes().get(0).getStorageType());
+ LOG.info("Disk Volume set {} - Type : {} plan completed.",
+ currentSet.getSetID(),
+ currentSet.getVolumes().get(0).getStorageType());
plan.setNodeName(node.getDataNodeName());
plan.setNodeUUID(node.getDataNodeUUID());
plan.setTimeStamp(Time.now());
plan.setPort(node.getDataNodePort());
- LOG.info(message);
}
/**
@@ -207,7 +203,7 @@ public class GreedyPlanner implements Planner {
// Create a new step
nextStep = new MoveStep(highVolume, currentSet.getIdealUsed(), lowVolume,
bytesToMove, currentSet.getSetID());
- LOG.debug(nextStep.toString());
+ LOG.debug("Next Step: {}", nextStep);
}
return nextStep;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HADOOP-15966. Hadoop Kerberos
broken on macos as java.security.krb5.realm is reset.
Posted by sh...@apache.org.
HADOOP-15966. Hadoop Kerberos broken on macos as java.security.krb5.realm is reset.
Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db2d8b01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db2d8b01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db2d8b01
Branch: refs/heads/HDFS-12943
Commit: db2d8b01c65fdff3acf83b4c4f0a79fe48270487
Parents: 7274115
Author: Steve Loughran <st...@apache.org>
Authored: Tue Dec 4 15:35:43 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Dec 4 15:35:43 2018 +0000
----------------------------------------------------------------------
.../hadoop-common/src/main/conf/hadoop-env.sh | 13 -------------
1 file changed, 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db2d8b01/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index 64d6bcb..e43cd95 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -96,19 +96,6 @@
# section as to why....
export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
-
-# Under certain conditions, Java on OS X will throw SCDynamicStore errors
-# in the system logs.
-# See HADOOP-8719 for more information. If one needs Kerberos
-# support on OS X, one will want to change/remove this extra bit.
-case ${HADOOP_OS_TYPE} in
- Darwin*)
- export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.realm= "
- export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.kdc= "
- export HADOOP_OPTS="${HADOOP_OPTS} -Djava.security.krb5.conf= "
- ;;
-esac
-
# Extra Java runtime options for some Hadoop commands
# and clients (i.e., hdfs dfs -blah). These get appended to HADOOP_OPTS for
# such commands. In most cases, # this should be left empty and
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-9036. Escape newlines in
health report in YARN UI. Contributed by Keqiu Hu
Posted by sh...@apache.org.
YARN-9036. Escape newlines in health report in YARN UI. Contributed by Keqiu Hu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d7b44c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d7b44c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d7b44c4
Branch: refs/heads/HDFS-12943
Commit: 6d7b44c48923c3e8a9c4d38f6803664dd5d61c2a
Parents: 38ea381
Author: Jonathan Hung <jh...@jhung-ld3.linkedin.biz>
Authored: Thu Nov 29 17:02:07 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Nov 30 10:13:52 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/resourcemanager/webapp/NodesPage.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d7b44c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 09b1c0f..cb92baf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import com.google.inject.Inject;
+import org.apache.commons.text.StringEscapeUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -203,8 +204,9 @@ class NodesPage extends RmView {
nodeTableData.length() - 1);
}
nodeTableData.append("]");
+ String nodeTableDataEscaped = StringEscapeUtils.escapeJava(nodeTableData.toString());
html.script().$type("text/javascript")
- .__("var nodeTableData=" + nodeTableData).__();
+ .__("var nodeTableData=" + nodeTableDataEscaped).__();
tbody.__().__();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDDS-848. Create SCM metrics
related to container state. Contributed by Bharat Viswanadham.
Posted by sh...@apache.org.
HDDS-848. Create SCM metrics related to container state. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3044b78b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3044b78b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3044b78b
Branch: refs/heads/HDFS-12943
Commit: 3044b78bd0191883d5f9daf2601a58a268beed06
Parents: 042c8ef
Author: Yiqun Lin <yq...@apache.org>
Authored: Mon Dec 3 17:16:34 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Mon Dec 3 17:16:34 2018 +0800
----------------------------------------------------------------------
.../hadoop/hdds/scm/server/SCMMXBean.java | 5 ++
.../scm/server/StorageContainerManager.java | 11 +++
.../apache/hadoop/ozone/scm/TestSCMMXBean.java | 81 ++++++++++++++++++--
3 files changed, 91 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044b78b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
index 4093918..dc09ceb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java
@@ -59,4 +59,9 @@ public interface SCMMXBean extends ServiceRuntimeInfo {
* @return String
*/
double getChillModeCurrentContainerThreshold();
+
+ /**
+ * Returns the container count in all states.
+ */
+ Map<String, Integer> getContainerStateCount();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044b78b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2d27984..a0d5e1d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -30,6 +30,7 @@ import com.google.protobuf.BlockingService;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.block.BlockManager;
@@ -925,6 +926,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return scmChillModeManager.getCurrentContainerThreshold();
}
+ @Override
+ public Map<String, Integer> getContainerStateCount() {
+ Map<String, Integer> nodeStateCount = new HashMap<>();
+ for (HddsProtos.LifeCycleState state: HddsProtos.LifeCycleState.values()) {
+ nodeStateCount.put(state.toString(), containerManager.getContainers(
+ state).size());
+ }
+ return nodeStateCount;
+ }
+
/**
* Startup options.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044b78b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMXBean.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMXBean.java
index 3136df2..eabf5e0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMXBean.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMXBean.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.ozone.scm;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -31,8 +35,11 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Iterator;
+import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeoutException;
@@ -118,16 +125,77 @@ public class TestSCMMXBean {
assertEquals(scm.getCurrentContainerThreshold(), containerThreshold, 0);
}
+ @Test
+ public void testSCMContainerStateCount() throws Exception {
+
+ ObjectName bean = new ObjectName(
+ "Hadoop:service=StorageContainerManager,"
+ + "name=StorageContainerManagerInfo,"
+ + "component=ServerRuntime");
+ TabularData data = (TabularData) mbs.getAttribute(
+ bean, "ContainerStateCount");
+ Map<String, Integer> containerStateCount = scm.getContainerStateCount();
+ verifyEquals(data, containerStateCount);
+
+ // Do some changes like allocate containers and change the container states
+ ContainerManager scmContainerManager = scm.getContainerManager();
+
+ List<ContainerInfo> containerInfoList = new ArrayList<>();
+ for (int i=0; i < 10; i++) {
+ containerInfoList.add(scmContainerManager.allocateContainer(HddsProtos
+ .ReplicationType.STAND_ALONE, HddsProtos.ReplicationFactor.ONE,
+ UUID.randomUUID().toString()));
+ }
+ long containerID;
+ for (int i=0; i < 10; i++) {
+ if (i % 2 == 0) {
+ containerID = containerInfoList.get(i).getContainerID();
+ scmContainerManager.updateContainerState(
+ new ContainerID(containerID), HddsProtos.LifeCycleEvent.FINALIZE);
+ assertEquals(scmContainerManager.getContainer(new ContainerID(
+ containerID)).getState(), HddsProtos.LifeCycleState.CLOSING);
+ } else {
+ containerID = containerInfoList.get(i).getContainerID();
+ scmContainerManager.updateContainerState(
+ new ContainerID(containerID), HddsProtos.LifeCycleEvent.FINALIZE);
+ scmContainerManager.updateContainerState(
+ new ContainerID(containerID), HddsProtos.LifeCycleEvent.CLOSE);
+ assertEquals(scmContainerManager.getContainer(new ContainerID(
+ containerID)).getState(), HddsProtos.LifeCycleState.CLOSED);
+ }
+
+ }
+
+ data = (TabularData) mbs.getAttribute(
+ bean, "ContainerStateCount");
+ containerStateCount = scm.getContainerStateCount();
+
+ containerStateCount.forEach((k, v) -> {
+ if(k == HddsProtos.LifeCycleState.CLOSING.toString()) {
+ assertEquals((int)v, 5);
+ } else if (k == HddsProtos.LifeCycleState.CLOSED.toString()) {
+ assertEquals((int)v, 5);
+ } else {
+ // Remaining all container state count should be zero.
+ assertEquals((int)v, 0);
+ }
+ });
+
+ verifyEquals(data, containerStateCount);
+
+ }
+
+
/**
* An internal function used to compare a TabularData returned
* by JMX with the expected data in a Map.
*/
- private void verifyEquals(TabularData data1,
- Map<String, Integer> data2) {
- if (data1 == null || data2 == null) {
+ private void verifyEquals(TabularData actualData,
+ Map<String, Integer> expectedData) {
+ if (actualData == null || expectedData == null) {
fail("Data should not be null.");
}
- for (Object obj : data1.values()) {
+ for (Object obj : actualData.values()) {
// Each TabularData is a set of CompositeData
assertTrue(obj instanceof CompositeData);
CompositeData cds = (CompositeData) obj;
@@ -136,8 +204,9 @@ public class TestSCMMXBean {
String key = it.next().toString();
String value = it.next().toString();
int num = Integer.parseInt(value);
- assertTrue(data2.containsKey(key));
- assertEquals(data2.get(key).intValue(), num);
+ assertTrue(expectedData.containsKey(key));
+ assertEquals(expectedData.remove(key).intValue(), num);
}
+ assertTrue(expectedData.isEmpty());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDDS-890. Handle
OverlappingFileLockException during writeStateMachineData in
ContainerStateMachine. Contributed by Shashikant Banerjee.
Posted by sh...@apache.org.
HDDS-890. Handle OverlappingFileLockException during writeStateMachineData in ContainerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7274115d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7274115d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7274115d
Branch: refs/heads/HDFS-12943
Commit: 7274115d57fdfef48fca1afa7be7ed2634dd31fa
Parents: ff31313
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Tue Dec 4 20:12:35 2018 +0530
Committer: Mukul Kumar Singh <ms...@apache.org>
Committed: Tue Dec 4 20:12:35 2018 +0530
----------------------------------------------------------------------
.../container/keyvalue/KeyValueHandler.java | 36 ++++++++++--------
.../keyvalue/impl/ChunkManagerImpl.java | 33 +++++++++-------
.../keyvalue/interfaces/ChunkManager.java | 8 ++--
.../keyvalue/TestChunkManagerImpl.java | 40 ++++++++++++--------
.../common/impl/TestContainerPersistence.java | 38 +++++++++++--------
5 files changed, 93 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7274115d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 5130253..01964ba 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -545,10 +545,12 @@ public class KeyValueHandler extends Handler {
.getChunkData());
Preconditions.checkNotNull(chunkInfo);
- boolean isReadFromTmpFile = dispatcherContext == null ? false :
- dispatcherContext.isReadFromTmpFile();
+ if (dispatcherContext == null) {
+ dispatcherContext = new DispatcherContext.Builder().build();
+ }
+
data = chunkManager
- .readChunk(kvContainer, blockID, chunkInfo, isReadFromTmpFile);
+ .readChunk(kvContainer, blockID, chunkInfo, dispatcherContext);
metrics.incContainerBytesStats(Type.ReadChunk, data.length);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -619,15 +621,17 @@ public class KeyValueHandler extends Handler {
Preconditions.checkNotNull(chunkInfo);
ByteBuffer data = null;
- WriteChunkStage stage =
- dispatcherContext == null ? WriteChunkStage.COMBINED :
- dispatcherContext.getStage();
+ if (dispatcherContext == null) {
+ dispatcherContext = new DispatcherContext.Builder().build();
+ }
+ WriteChunkStage stage = dispatcherContext.getStage();
if (stage == WriteChunkStage.WRITE_DATA ||
stage == WriteChunkStage.COMBINED) {
data = request.getWriteChunk().getData().asReadOnlyByteBuffer();
}
- chunkManager.writeChunk(kvContainer, blockID, chunkInfo, data, stage);
+ chunkManager
+ .writeChunk(kvContainer, blockID, chunkInfo, data, dispatcherContext);
// We should increment stats after writeChunk
if (stage == WriteChunkStage.WRITE_DATA||
@@ -677,19 +681,19 @@ public class KeyValueHandler extends Handler {
putSmallFileReq.getChunkInfo());
Preconditions.checkNotNull(chunkInfo);
ByteBuffer data = putSmallFileReq.getData().asReadOnlyByteBuffer();
- WriteChunkStage stage =
- dispatcherContext == null ? WriteChunkStage.COMBINED :
- dispatcherContext.getStage();
+ if (dispatcherContext == null) {
+ dispatcherContext = new DispatcherContext.Builder().build();
+ }
+
// chunks will be committed as a part of handling putSmallFile
// here. There is no need to maintain this info in openContainerBlockMap.
- chunkManager.writeChunk(kvContainer, blockID, chunkInfo, data, stage);
+ chunkManager
+ .writeChunk(kvContainer, blockID, chunkInfo, data, dispatcherContext);
List<ContainerProtos.ChunkInfo> chunks = new LinkedList<>();
chunks.add(chunkInfo.getProtoBufMessage());
blockData.setChunks(chunks);
- long bcsId =
- dispatcherContext == null ? 0 : dispatcherContext.getLogIndex();
- blockData.setBlockCommitSequenceId(bcsId);
+ blockData.setBlockCommitSequenceId(dispatcherContext.getLogIndex());
blockManager.putBlock(kvContainer, blockData);
metrics.incContainerBytesStats(Type.PutSmallFile, data.capacity());
@@ -728,11 +732,13 @@ public class KeyValueHandler extends Handler {
ContainerProtos.ChunkInfo chunkInfo = null;
ByteString dataBuf = ByteString.EMPTY;
+ DispatcherContext dispatcherContext =
+ new DispatcherContext.Builder().build();
for (ContainerProtos.ChunkInfo chunk : responseData.getChunks()) {
// if the block is committed, all chunks must have been committed.
// Tmp chunk files won't exist here.
byte[] data = chunkManager.readChunk(kvContainer, blockID,
- ChunkInfo.getFromProtoBuf(chunk), false);
+ ChunkInfo.getFromProtoBuf(chunk), dispatcherContext);
ByteString current = ByteString.copyFrom(data);
dataBuf = dataBuf.concat(current);
chunkInfo = chunk;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7274115d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index a2e8e5c..e4814cb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -67,13 +67,14 @@ public class ChunkManagerImpl implements ChunkManager {
* @param blockID - ID of the block
* @param info - ChunkInfo
* @param data - data of the chunk
- * @param stage - Stage of the Chunk operation
+ * @param dispatcherContext - dispatcherContextInfo
* @throws StorageContainerException
*/
public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
- ByteBuffer data, DispatcherContext.WriteChunkStage stage)
+ ByteBuffer data, DispatcherContext dispatcherContext)
throws StorageContainerException {
-
+ Preconditions.checkNotNull(dispatcherContext);
+ DispatcherContext.WriteChunkStage stage = dispatcherContext.getStage();
try {
KeyValueContainerData containerData = (KeyValueContainerData) container
@@ -85,7 +86,7 @@ public class ChunkManagerImpl implements ChunkManager {
boolean isOverwrite = ChunkUtils.validateChunkForOverwrite(
chunkFile, info);
- File tmpChunkFile = getTmpChunkFile(chunkFile, info);
+ File tmpChunkFile = getTmpChunkFile(chunkFile, dispatcherContext);
LOG.debug(
"writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file:{}",
@@ -137,6 +138,8 @@ public class ChunkManagerImpl implements ChunkManager {
LOG.warn("ChunkFile already exists" + chunkFile);
return;
}
+ // While committing a chunk , just rename the tmp chunk file which has
+ // the same term and log index appended as the current transaction
commitChunk(tmpChunkFile, chunkFile);
// Increment container stats here, as we commit the data.
containerData.incrBytesUsed(info.getLen());
@@ -179,14 +182,14 @@ public class ChunkManagerImpl implements ChunkManager {
* @param container - Container for the chunk
* @param blockID - ID of the block.
* @param info - ChunkInfo.
- * @param readFromTmpFile whether to read from tmp chunk file or not.
+ * @param dispatcherContext dispatcher context info.
* @return byte array
* @throws StorageContainerException
* TODO: Right now we do not support partial reads and writes of chunks.
* TODO: Explore if we need to do that for ozone.
*/
public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
- boolean readFromTmpFile) throws StorageContainerException {
+ DispatcherContext dispatcherContext) throws StorageContainerException {
try {
KeyValueContainerData containerData = (KeyValueContainerData) container
.getContainerData();
@@ -204,8 +207,8 @@ public class ChunkManagerImpl implements ChunkManager {
// In case the chunk file does not exist but tmp chunk file exist,
// read from tmp chunk file if readFromTmpFile is set to true
- if (!chunkFile.exists() && readFromTmpFile) {
- chunkFile = getTmpChunkFile(chunkFile, info);
+ if (!chunkFile.exists() && dispatcherContext.isReadFromTmpFile()) {
+ chunkFile = getTmpChunkFile(chunkFile, dispatcherContext);
}
data = ChunkUtils.readData(chunkFile, info, volumeIOStats);
containerData.incrReadCount();
@@ -279,17 +282,21 @@ public class ChunkManagerImpl implements ChunkManager {
/**
* Returns the temporary chunkFile path.
- * @param chunkFile
- * @param info
+ * @param chunkFile chunkFileName
+ * @param dispatcherContext dispatcher context info
* @return temporary chunkFile path
* @throws StorageContainerException
*/
- private File getTmpChunkFile(File chunkFile, ChunkInfo info)
- throws StorageContainerException {
+ private File getTmpChunkFile(File chunkFile,
+ DispatcherContext dispatcherContext) {
return new File(chunkFile.getParent(),
chunkFile.getName() +
OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
- OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX);
+ OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX +
+ OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
+ dispatcherContext.getTerm() +
+ OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
+ dispatcherContext.getLogIndex());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7274115d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index 4282e46..5a6898f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -39,11 +39,11 @@ public interface ChunkManager {
* @param container - Container for the chunk
* @param blockID - ID of the block.
* @param info - ChunkInfo.
- * @param stage - Chunk Stage write.
+ * @param dispatcherContext - dispatcher context info.
* @throws StorageContainerException
*/
void writeChunk(Container container, BlockID blockID, ChunkInfo info,
- ByteBuffer data, DispatcherContext.WriteChunkStage stage)
+ ByteBuffer data, DispatcherContext dispatcherContext)
throws StorageContainerException;
/**
@@ -52,7 +52,7 @@ public interface ChunkManager {
* @param container - Container for the chunk
* @param blockID - ID of the block.
* @param info - ChunkInfo.
- * @param readFromTmpFile whether to read from tmp chunk file or not
+ * @param dispatcherContext - dispatcher context info.
* @return byte array
* @throws StorageContainerException
*
@@ -60,7 +60,7 @@ public interface ChunkManager {
* TODO: Explore if we need to do that for ozone.
*/
byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
- boolean readFromTmpFile) throws StorageContainerException;
+ DispatcherContext dispatcherContext) throws StorageContainerException;
/**
* Deletes a given chunk.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7274115d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index fd48bf5..cf9ea89 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext.WriteChunkStage;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
@@ -103,6 +103,10 @@ public class TestChunkManagerImpl {
}
+ private DispatcherContext getDispatcherContext() {
+ return new DispatcherContext.Builder().build();
+ }
+
@Test
public void testWriteChunkStageWriteAndCommit() throws Exception {
//As in Setup, we try to create container, these paths should exist.
@@ -115,16 +119,20 @@ public class TestChunkManagerImpl {
// As no chunks are written to the volume writeBytes should be 0
checkWriteIOStats(0, 0);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.WRITE_DATA);
+ ByteBuffer.wrap(data), new DispatcherContext.Builder()
+ .setStage(DispatcherContext.WriteChunkStage.WRITE_DATA).build());
// Now a chunk file is being written with Stage WRITE_DATA, so it should
// create a temporary chunk file.
assertTrue(chunksPath.listFiles().length == 1);
+ long term = 0;
+ long index = 0;
File chunkFile = ChunkUtils.getChunkFile(keyValueContainerData, chunkInfo);
File tempChunkFile = new File(chunkFile.getParent(),
- chunkFile.getName() +
- OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
- OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX);
+ chunkFile.getName() + OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER
+ + OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX
+ + OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER + term
+ + OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER + index);
// As chunk write stage is WRITE_DATA, temp chunk file will be created.
assertTrue(tempChunkFile.exists());
@@ -132,7 +140,8 @@ public class TestChunkManagerImpl {
checkWriteIOStats(data.length, 1);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.COMMIT_DATA);
+ ByteBuffer.wrap(data), new DispatcherContext.Builder()
+ .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA).build());
checkWriteIOStats(data.length, 1);
@@ -152,7 +161,7 @@ public class TestChunkManagerImpl {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, randomLength);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.WRITE_DATA);
+ ByteBuffer.wrap(data), getDispatcherContext());
fail("testWriteChunkIncorrectLength failed");
} catch (StorageContainerException ex) {
// As we got an exception, writeBytes should be 0.
@@ -173,7 +182,7 @@ public class TestChunkManagerImpl {
assertTrue(chunksPath.listFiles().length == 0);
checkWriteIOStats(0, 0);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
+ ByteBuffer.wrap(data), getDispatcherContext());
// Now a chunk file is being written with Stage COMBINED_DATA, so it should
// create a chunk file.
assertTrue(chunksPath.listFiles().length == 1);
@@ -186,11 +195,11 @@ public class TestChunkManagerImpl {
public void testReadChunk() throws Exception {
checkWriteIOStats(0, 0);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
+ ByteBuffer.wrap(data), getDispatcherContext());
checkWriteIOStats(data.length, 1);
checkReadIOStats(0, 0);
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
- chunkInfo, false);
+ chunkInfo, getDispatcherContext());
assertEquals(expectedData.length, data.length);
assertTrue(Arrays.equals(expectedData, data));
checkReadIOStats(data.length, 1);
@@ -200,7 +209,7 @@ public class TestChunkManagerImpl {
public void testDeleteChunk() throws Exception {
File chunksPath = new File(keyValueContainerData.getChunksPath());
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
+ ByteBuffer.wrap(data), getDispatcherContext());
assertTrue(chunksPath.listFiles().length == 1);
chunkManager.deleteChunk(keyValueContainer, blockID, chunkInfo);
assertTrue(chunksPath.listFiles().length == 0);
@@ -210,7 +219,7 @@ public class TestChunkManagerImpl {
public void testDeleteChunkUnsupportedRequest() throws Exception {
try {
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
+ ByteBuffer.wrap(data), getDispatcherContext());
long randomLength = 200L;
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, randomLength);
@@ -227,7 +236,7 @@ public class TestChunkManagerImpl {
try {
// trying to read a chunk, where chunk file does not exist
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
- chunkInfo, false);
+ chunkInfo, getDispatcherContext());
fail("testReadChunkFileNotExists failed");
} catch (StorageContainerException ex) {
GenericTestUtils.assertExceptionContains("Unable to find the chunk " +
@@ -242,7 +251,7 @@ public class TestChunkManagerImpl {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), i), 0, data.length);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
+ ByteBuffer.wrap(data), getDispatcherContext());
}
checkWriteIOStats(data.length*100, 100);
assertTrue(hddsVolume.getVolumeIOStats().getWriteTime() > 0);
@@ -250,7 +259,8 @@ public class TestChunkManagerImpl {
for (int i=0; i<100; i++) {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), i), 0, data.length);
- chunkManager.readChunk(keyValueContainer, blockID, chunkInfo, false);
+ chunkManager.readChunk(keyValueContainer, blockID, chunkInfo,
+ getDispatcherContext());
}
checkReadIOStats(data.length*100, 100);
assertTrue(hddsVolume.getVolumeIOStats().getReadTime() > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7274115d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 32b01ae..d89ffb6 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
@@ -78,7 +79,6 @@ import java.util.UUID;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.BCSID_MISMATCH;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNKNOWN_BCSID;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext.WriteChunkStage;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getData;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.setDataChecksum;
@@ -156,6 +156,10 @@ public class TestContainerPersistence {
return ContainerTestHelper.getTestContainerID();
}
+ private DispatcherContext getDispatcherContext() {
+ return new DispatcherContext.Builder().build();
+ }
+
private Container addContainer(ContainerSet cSet, long cID)
throws IOException {
KeyValueContainerData data = new KeyValueContainerData(cID,
@@ -334,7 +338,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
return info;
}
@@ -375,7 +379,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
String fileName = String.format("%s.data.%d", blockID.getLocalID(), x);
fileHashMap.put(fileName, info);
}
@@ -406,7 +410,8 @@ public class TestContainerPersistence {
for (int x = 0; x < chunkCount; x++) {
String fileName = String.format("%s.data.%d", blockID.getLocalID(), x);
ChunkInfo info = fileHashMap.get(fileName);
- byte[] data = chunkManager.readChunk(container, blockID, info, false);
+ byte[] data = chunkManager
+ .readChunk(container, blockID, info, getDispatcherContext());
ChecksumData checksumData = checksum.computeChecksum(data);
Assert.assertEquals(info.getChecksumData(), checksumData);
}
@@ -433,13 +438,15 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
- byte[] readData = chunkManager.readChunk(container, blockID, info, false);
+ byte[] readData = chunkManager
+ .readChunk(container, blockID, info, getDispatcherContext());
assertTrue(Arrays.equals(data, readData));
ChunkInfo info2 = getChunk(blockID.getLocalID(), 0, start, length);
- byte[] readData2 = chunkManager.readChunk(container, blockID, info2, false);
+ byte[] readData2 = chunkManager
+ .readChunk(container, blockID, info2, getDispatcherContext());
assertEquals(length, readData2.length);
assertTrue(Arrays.equals(
Arrays.copyOfRange(data, start, start + length), readData2));
@@ -466,13 +473,13 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
// With the overwrite flag it should work now.
info.addMetadata(OzoneConsts.CHUNK_OVERWRITE, "true");
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
long bytesUsed = container.getContainerData().getBytesUsed();
Assert.assertEquals(datalen, bytesUsed);
@@ -507,14 +514,15 @@ public class TestContainerPersistence {
oldSha.update(data);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
}
// Request to read the whole data in a single go.
ChunkInfo largeChunk = getChunk(blockID.getLocalID(), 0, 0,
datalen * chunkCount);
byte[] newdata =
- chunkManager.readChunk(container, blockID, largeChunk, false);
+ chunkManager.readChunk(container, blockID, largeChunk,
+ getDispatcherContext());
MessageDigest newSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
newSha.update(newdata);
Assert.assertEquals(Hex.encodeHexString(oldSha.digest()),
@@ -540,11 +548,11 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
chunkManager.deleteChunk(container, blockID, info);
exception.expect(StorageContainerException.class);
exception.expectMessage("Unable to find the chunk file.");
- chunkManager.readChunk(container, blockID, info, false);
+ chunkManager.readChunk(container, blockID, info, getDispatcherContext());
}
/**
@@ -655,7 +663,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- WriteChunkStage.COMBINED);
+ getDispatcherContext());
totalSize += datalen;
chunkList.add(info);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDDS-748. Use strongly typed
metadata Table implementation. Contributed by Elek Marton.
Posted by sh...@apache.org.
HDDS-748. Use strongly typed metadata Table implementation. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d15dc436
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d15dc436
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d15dc436
Branch: refs/heads/HDFS-12943
Commit: d15dc436598d646de67b553207ab6624741f56a5
Parents: 99e201d
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Sat Dec 1 16:52:23 2018 -0800
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Sat Dec 1 16:52:23 2018 -0800
----------------------------------------------------------------------
.../hadoop/utils/db/ByteArrayKeyValue.java | 67 ++++++
.../java/org/apache/hadoop/utils/db/Codec.java | 38 +++
.../apache/hadoop/utils/db/CodecRegistry.java | 70 ++++++
.../org/apache/hadoop/utils/db/DBStore.java | 12 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 12 +-
.../hadoop/utils/db/RDBStoreIterator.java | 20 +-
.../org/apache/hadoop/utils/db/RDBTable.java | 7 +-
.../org/apache/hadoop/utils/db/StringCodec.java | 45 ++++
.../java/org/apache/hadoop/utils/db/Table.java | 63 +----
.../apache/hadoop/utils/db/TableIterator.java | 6 +-
.../org/apache/hadoop/utils/db/TypedTable.java | 184 +++++++++++++++
.../hadoop/utils/db/TestDBStoreBuilder.java | 4 +-
.../apache/hadoop/utils/db/TestRDBStore.java | 31 +--
.../hadoop/utils/db/TestRDBTableStore.java | 4 +-
.../hadoop/utils/db/TestTypedRDBTableStore.java | 235 +++++++++++++++++++
.../hadoop/ozone/om/OMMetadataManager.java | 17 +-
.../hadoop/ozone/om/TestOzoneManager.java | 4 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 82 ++++---
18 files changed, 765 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/ByteArrayKeyValue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/ByteArrayKeyValue.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/ByteArrayKeyValue.java
new file mode 100644
index 0000000..ca5583c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/ByteArrayKeyValue.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.utils.db;
+
+import org.apache.hadoop.utils.db.Table.KeyValue;
+
+/**
+ * Key value for raw Table implementations.
+ */
+public final class ByteArrayKeyValue implements KeyValue<byte[], byte[]> {
+ private byte[] key;
+ private byte[] value;
+
+ private ByteArrayKeyValue(byte[] key, byte[] value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ /**
+ * Create a KeyValue pair.
+ *
+ * @param key - Key Bytes
+ * @param value - Value bytes
+ * @return KeyValue object.
+ */
+ public static ByteArrayKeyValue create(byte[] key, byte[] value) {
+ return new ByteArrayKeyValue(key, value);
+ }
+
+ /**
+ * Return key.
+ *
+ * @return byte[]
+ */
+ public byte[] getKey() {
+ byte[] result = new byte[key.length];
+ System.arraycopy(key, 0, result, 0, key.length);
+ return result;
+ }
+
+ /**
+ * Return value.
+ *
+ * @return byte[]
+ */
+ public byte[] getValue() {
+ byte[] result = new byte[value.length];
+ System.arraycopy(value, 0, result, 0, value.length);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
new file mode 100644
index 0000000..7f6f489
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.utils.db;
+
+/**
+ * Codec interface to marshall/unmarshall data to/from a byte[] based
+ * key/value store.
+ *
+ * @param <T> Unserialized type
+ */
+public interface Codec<T> {
+
+ /**
+ * Convert object to raw persisted format.
+ */
+ byte[] toPersistedFormat(T object);
+
+ /**
+ * Convert object from raw persisted format.
+ */
+ T fromPersistedFormat(byte[] rawData);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
new file mode 100644
index 0000000..9bdff19
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.utils.db;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Collection of available codecs.
+ */
+public class CodecRegistry {
+
+ private Map<Class, Codec<?>> valueCodecs;
+
+ public CodecRegistry() {
+ valueCodecs = new HashMap<>();
+ valueCodecs.put(String.class, new StringCodec());
+ }
+
+ /**
+ * Convert raw value to strongly typed value/key with the help of a codec.
+ *
+ * @param rawData original byte array from the db.
+ * @param format Class of the return value
+ * @param <T> Type of the return value.
+ * @return the object with the parsed field data
+ */
+ public <T> T asObject(byte[] rawData, Class<T> format) {
+ if (valueCodecs.containsKey(format)) {
+ return (T) valueCodecs.get(format).fromPersistedFormat(rawData);
+ } else {
+ throw new IllegalStateException(
+ "Codec is not registered for type: " + format);
+ }
+ }
+
+ /**
+ * Convert strongly typed object to raw data to store it in the kv store.
+ *
+ * @param object typed object.
+ * @param <T> Type of the typed object.
+ * @return byte array to store it ini the kv store.
+ */
+ public <T> byte[] asRawData(T object) {
+ Class<T> format = (Class<T>) object.getClass();
+ if (valueCodecs.containsKey(format)) {
+ Codec<T> codec = (Codec<T>) valueCodecs.get(format);
+ return codec.toPersistedFormat(object);
+ } else {
+ throw new IllegalStateException(
+ "Codec is not registered for type: " + format);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
index 26dbc43..51b716b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
@@ -41,7 +41,17 @@ public interface DBStore extends AutoCloseable {
* @return - TableStore.
* @throws IOException on Failure
*/
- Table getTable(String name) throws IOException;
+ Table<byte[], byte[]> getTable(String name) throws IOException;
+
+ /**
+ * Gets an existing TableStore with implicit key/value conversion.
+ *
+ * @param name - Name of the TableStore to get
+ * @return - TableStore.
+ * @throws IOException on Failure
+ */
+ <KEY, VALUE> Table<KEY, VALUE> getTable(String name,
+ Class<KEY> keyType, Class<VALUE> valueType) throws IOException;
/**
* Lists the Known list of Tables in a DB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
index 68c35fa..8ed60ad 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
@@ -55,6 +55,7 @@ public class RDBStore implements DBStore {
private final File dbLocation;
private final WriteOptions writeOptions;
private final DBOptions dbOptions;
+ private final CodecRegistry codecRegistry;
private final Hashtable<String, ColumnFamilyHandle> handleTable;
private ObjectName statMBeanName;
@@ -64,7 +65,7 @@ public class RDBStore implements DBStore {
Preconditions.checkNotNull(families);
Preconditions.checkArgument(families.size() > 0);
handleTable = new Hashtable<>();
-
+ codecRegistry = new CodecRegistry();
final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
new ArrayList<>();
final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
@@ -254,7 +255,7 @@ public class RDBStore implements DBStore {
}
@Override
- public Table getTable(String name) throws IOException {
+ public Table<byte[], byte[]> getTable(String name) throws IOException {
ColumnFamilyHandle handle = handleTable.get(name);
if (handle == null) {
throw new IOException("No such table in this DB. TableName : " + name);
@@ -263,6 +264,13 @@ public class RDBStore implements DBStore {
}
@Override
+ public <KEY, VALUE> Table<KEY, VALUE> getTable(String name,
+ Class<KEY> keyType, Class<VALUE> valueType) throws IOException {
+ return new TypedTable<KEY, VALUE>(getTable(name), codecRegistry, keyType,
+ valueType);
+ }
+
+ @Override
public ArrayList<Table> listTables() throws IOException {
ArrayList<Table> returnList = new ArrayList<>();
for (ColumnFamilyHandle handle : handleTable.values()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java
index f1f2df6..1faa089 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java
@@ -19,17 +19,17 @@
package org.apache.hadoop.utils.db;
-import org.apache.hadoop.utils.db.Table.KeyValue;
-import org.rocksdb.RocksIterator;
-
import java.io.IOException;
import java.util.NoSuchElementException;
import java.util.function.Consumer;
+import org.rocksdb.RocksIterator;
+
/**
* RocksDB store iterator.
*/
-public class RDBStoreIterator implements TableIterator<KeyValue> {
+public class RDBStoreIterator
+ implements TableIterator<byte[], ByteArrayKeyValue> {
private RocksIterator rocksDBIterator;
@@ -39,7 +39,8 @@ public class RDBStoreIterator implements TableIterator<KeyValue> {
}
@Override
- public void forEachRemaining(Consumer<? super KeyValue> action) {
+ public void forEachRemaining(
+ Consumer<? super ByteArrayKeyValue> action) {
while(hasNext()) {
action.accept(next());
}
@@ -51,9 +52,10 @@ public class RDBStoreIterator implements TableIterator<KeyValue> {
}
@Override
- public Table.KeyValue next() {
+ public ByteArrayKeyValue next() {
if (rocksDBIterator.isValid()) {
- KeyValue value = KeyValue.create(rocksDBIterator.key(), rocksDBIterator
+ ByteArrayKeyValue value =
+ ByteArrayKeyValue.create(rocksDBIterator.key(), rocksDBIterator
.value());
rocksDBIterator.next();
return value;
@@ -72,10 +74,10 @@ public class RDBStoreIterator implements TableIterator<KeyValue> {
}
@Override
- public KeyValue seek(byte[] key) {
+ public ByteArrayKeyValue seek(byte[] key) {
rocksDBIterator.seek(key);
if (rocksDBIterator.isValid()) {
- return KeyValue.create(rocksDBIterator.key(),
+ return ByteArrayKeyValue.create(rocksDBIterator.key(),
rocksDBIterator.value());
}
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java
index 6b504c2..88b0411 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java
@@ -35,7 +35,8 @@ import org.slf4j.LoggerFactory;
/**
* RocksDB implementation of ozone metadata store.
*/
-public class RDBTable implements Table {
+public class RDBTable implements Table<byte[], byte[]> {
+
private static final Logger LOG =
LoggerFactory.getLogger(RDBTable.class);
@@ -108,7 +109,7 @@ public class RDBTable implements Table {
@Override
public boolean isEmpty() throws IOException {
- try (TableIterator<KeyValue> keyIter = iterator()) {
+ try (TableIterator<byte[], ByteArrayKeyValue> keyIter = iterator()) {
keyIter.seekToFirst();
return !keyIter.hasNext();
}
@@ -145,7 +146,7 @@ public class RDBTable implements Table {
}
@Override
- public TableIterator<KeyValue> iterator() {
+ public TableIterator<byte[], ByteArrayKeyValue> iterator() {
ReadOptions readOptions = new ReadOptions();
return new RDBStoreIterator(db.newIterator(handle, readOptions));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/StringCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/StringCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/StringCodec.java
new file mode 100644
index 0000000..36e4bc3
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/StringCodec.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.utils.db;
+
+import org.apache.hadoop.hdfs.DFSUtil;
+
+/**
+ * Codec to convert String to/from byte array.
+ */
+public class StringCodec implements Codec<String> {
+
+ @Override
+ public byte[] toPersistedFormat(String object) {
+ if (object != null) {
+ return DFSUtil.string2Bytes(object);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public String fromPersistedFormat(byte[] rawData) {
+ if (rawData != null) {
+ return DFSUtil.bytes2String(rawData);
+ } else {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java
index 8bbd247..e39c872 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* different kind of tables.
*/
@InterfaceStability.Evolving
-public interface Table extends AutoCloseable {
+public interface Table<KEY, VALUE> extends AutoCloseable {
/**
* Puts a key-value pair into the store.
@@ -38,7 +38,7 @@ public interface Table extends AutoCloseable {
* @param key metadata key
* @param value metadata value
*/
- void put(byte[] key, byte[] value) throws IOException;
+ void put(KEY key, VALUE value) throws IOException;
/**
* Puts a key-value pair into the store as part of a bath operation.
@@ -47,7 +47,7 @@ public interface Table extends AutoCloseable {
* @param key metadata key
* @param value metadata value
*/
- void putWithBatch(BatchOperation batch, byte[] key, byte[] value)
+ void putWithBatch(BatchOperation batch, KEY key, VALUE value)
throws IOException;
/**
@@ -64,7 +64,7 @@ public interface Table extends AutoCloseable {
* @return value in byte array or null if the key is not found.
* @throws IOException on Failure
*/
- byte[] get(byte[] key) throws IOException;
+ VALUE get(KEY key) throws IOException;
/**
* Deletes a key from the metadata store.
@@ -72,7 +72,7 @@ public interface Table extends AutoCloseable {
* @param key metadata key
* @throws IOException on Failure
*/
- void delete(byte[] key) throws IOException;
+ void delete(KEY key) throws IOException;
/**
* Deletes a key from the metadata store as part of a batch operation.
@@ -81,14 +81,14 @@ public interface Table extends AutoCloseable {
* @param key metadata key
* @throws IOException on Failure
*/
- void deleteWithBatch(BatchOperation batch, byte[] key) throws IOException;
+ void deleteWithBatch(BatchOperation batch, KEY key) throws IOException;
/**
* Returns the iterator for this metadata store.
*
* @return MetaStoreIterator
*/
- TableIterator<KeyValue> iterator();
+ TableIterator<KEY, ? extends KeyValue<KEY, VALUE>> iterator();
/**
* Returns the Name of this Table.
@@ -100,53 +100,10 @@ public interface Table extends AutoCloseable {
/**
* Class used to represent the key and value pair of a db entry.
*/
- class KeyValue {
+ interface KeyValue<KEY, VALUE> {
- private final byte[] key;
- private final byte[] value;
+ KEY getKey();
- /**
- * KeyValue Constructor, used to represent a key and value of a db entry.
- *
- * @param key - Key Bytes
- * @param value - Value bytes
- */
- private KeyValue(byte[] key, byte[] value) {
- this.key = key;
- this.value = value;
- }
-
- /**
- * Create a KeyValue pair.
- *
- * @param key - Key Bytes
- * @param value - Value bytes
- * @return KeyValue object.
- */
- public static KeyValue create(byte[] key, byte[] value) {
- return new KeyValue(key, value);
- }
-
- /**
- * Return key.
- *
- * @return byte[]
- */
- public byte[] getKey() {
- byte[] result = new byte[key.length];
- System.arraycopy(key, 0, result, 0, key.length);
- return result;
- }
-
- /**
- * Return value.
- *
- * @return byte[]
- */
- public byte[] getValue() {
- byte[] result = new byte[value.length];
- System.arraycopy(value, 0, result, 0, value.length);
- return result;
- }
+ VALUE getValue();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java
index 83a8f3c..071dbf4 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java
@@ -27,7 +27,7 @@ import java.util.Iterator;
*
* @param <T>
*/
-public interface TableIterator<T> extends Iterator<T>, Closeable {
+public interface TableIterator<KEY, T> extends Iterator<T>, Closeable {
/**
* seek to first entry.
@@ -43,8 +43,8 @@ public interface TableIterator<T> extends Iterator<T>, Closeable {
* Seek to the specific key.
*
* @param key - Bytes that represent the key.
- * @return T.
+ * @return VALUE.
*/
- T seek(byte[] key);
+ T seek(KEY key);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
new file mode 100644
index 0000000..087752c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.utils.db;
+
+import java.io.IOException;
+
+/**
+ * Strongly typed table implementation.
+ * <p>
+ * Automatically converts values and keys using a raw byte[] based table
+ * implementation and registered converters.
+ *
+ * @param <KEY> type of the keys in the store.
+ * @param <VALUE> type of the values in the store.
+ */
+public class TypedTable<KEY, VALUE> implements Table<KEY, VALUE> {
+
+ private Table<byte[], byte[]> rawTable;
+
+ private CodecRegistry codecRegistry;
+
+ private Class<KEY> keyType;
+
+ private Class<VALUE> valueType;
+
+ public TypedTable(
+ Table<byte[], byte[]> rawTable,
+ CodecRegistry codecRegistry, Class<KEY> keyType,
+ Class<VALUE> valueType) {
+ this.rawTable = rawTable;
+ this.codecRegistry = codecRegistry;
+ this.keyType = keyType;
+ this.valueType = valueType;
+ }
+
+ @Override
+ public void put(KEY key, VALUE value) throws IOException {
+ byte[] keyData = codecRegistry.asRawData(key);
+ byte[] valueData = codecRegistry.asRawData(value);
+ rawTable.put(keyData, valueData);
+ }
+
+ @Override
+ public void putWithBatch(BatchOperation batch, KEY key, VALUE value)
+ throws IOException {
+ byte[] keyData = codecRegistry.asRawData(key);
+ byte[] valueData = codecRegistry.asRawData(value);
+ rawTable.putWithBatch(batch, keyData, valueData);
+ }
+
+ @Override
+ public boolean isEmpty() throws IOException {
+ return rawTable.isEmpty();
+ }
+
+ @Override
+ public VALUE get(KEY key) throws IOException {
+ byte[] keyBytes = codecRegistry.asRawData(key);
+ byte[] valueBytes = rawTable.get(keyBytes);
+ return codecRegistry.asObject(valueBytes, valueType);
+ }
+
+ @Override
+ public void delete(KEY key) throws IOException {
+ rawTable.delete(codecRegistry.asRawData(key));
+ }
+
+ @Override
+ public void deleteWithBatch(BatchOperation batch, KEY key)
+ throws IOException {
+ rawTable.deleteWithBatch(batch, codecRegistry.asRawData(key));
+
+ }
+
+ @Override
+ public TableIterator<KEY, TypedKeyValue> iterator() {
+ TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> iterator =
+ rawTable.iterator();
+ return new TypedTableIterator(iterator, keyType, valueType);
+ }
+
+ @Override
+ public String getName() throws IOException {
+ return rawTable.getName();
+ }
+
+ @Override
+ public void close() throws Exception {
+ rawTable.close();
+
+ }
+
+ /**
+ * Key value implementation for strongly typed tables.
+ */
+ public class TypedKeyValue implements KeyValue<KEY, VALUE> {
+
+ private KeyValue<byte[], byte[]> rawKeyValue;
+
+ public TypedKeyValue(KeyValue<byte[], byte[]> rawKeyValue) {
+ this.rawKeyValue = rawKeyValue;
+ }
+
+ public TypedKeyValue(KeyValue<byte[], byte[]> rawKeyValue,
+ Class<KEY> keyType, Class<VALUE> valueType) {
+ this.rawKeyValue = rawKeyValue;
+ }
+
+ @Override
+ public KEY getKey() {
+ return codecRegistry.asObject(rawKeyValue.getKey(), keyType);
+ }
+
+ @Override
+ public VALUE getValue() {
+ return codecRegistry.asObject(rawKeyValue.getValue(), valueType);
+ }
+ }
+
+ /**
+ * Table Iterator implementation for strongly typed tables.
+ */
+ public class TypedTableIterator implements TableIterator<KEY, TypedKeyValue> {
+
+ private TableIterator<byte[], ? extends KeyValue<byte[], byte[]>>
+ rawIterator;
+
+ public TypedTableIterator(
+ TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> rawIterator,
+ Class<KEY> keyType,
+ Class<VALUE> valueType) {
+ this.rawIterator = rawIterator;
+ }
+
+ @Override
+ public void seekToFirst() {
+ rawIterator.seekToFirst();
+ }
+
+ @Override
+ public void seekToLast() {
+ rawIterator.seekToLast();
+ }
+
+ @Override
+ public TypedKeyValue seek(KEY key) {
+ byte[] keyBytes = codecRegistry.asRawData(key);
+ KeyValue<byte[], byte[]> result = rawIterator.seek(keyBytes);
+ return new TypedKeyValue(result);
+ }
+
+ @Override
+ public void close() throws IOException {
+ rawIterator.close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return rawIterator.hasNext();
+ }
+
+ @Override
+ public TypedKeyValue next() {
+ return new TypedKeyValue(rawIterator.next(), keyType,
+ valueType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
index 47ad597..993bcdf 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
@@ -124,7 +124,7 @@ public class TestDBStoreBuilder {
.addTable("First")
.addTable("Second")
.build()) {
- try (Table firstTable = dbStore.getTable("First")) {
+ try (Table<byte[], byte[]> firstTable = dbStore.getTable("First")) {
byte[] key =
RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
byte[] value =
@@ -154,7 +154,7 @@ public class TestDBStoreBuilder {
.addTable("Second")
.setProfile(DBProfile.DISK)
.build()) {
- try (Table firstTable = dbStore.getTable("First")) {
+ try (Table<byte[], byte[]> firstTable = dbStore.getTable("First")) {
byte[] key =
RandomStringUtils.random(9).getBytes(StandardCharsets.UTF_8);
byte[] value =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java
index 0b673d3..462d2e4 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java
@@ -19,8 +19,20 @@
package org.apache.hadoop.utils.db;
-import org.apache.commons.lang3.RandomStringUtils;
+import javax.management.MBeanServer;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.hdfs.DFSUtil;
+
+import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -34,17 +46,6 @@ import org.rocksdb.RocksDB;
import org.rocksdb.Statistics;
import org.rocksdb.StatsLevel;
-import javax.management.MBeanServer;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
/**
* RDBStore Tests.
*/
@@ -126,7 +127,8 @@ public class TestRDBStore {
try (Table firstTable = rdbStore.getTable(families.get(1))) {
firstTable.put(key, value);
- try (Table secondTable = rdbStore.getTable(families.get(2))) {
+ try (Table<byte[], byte[]> secondTable = rdbStore
+ .getTable(families.get(2))) {
rdbStore.move(key, firstTable, secondTable);
byte[] newvalue = secondTable.get(key);
// Make sure we have value in the second table
@@ -150,7 +152,8 @@ public class TestRDBStore {
RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8);
try (Table firstTable = rdbStore.getTable(families.get(1))) {
firstTable.put(key, value);
- try (Table secondTable = rdbStore.getTable(families.get(2))) {
+ try (Table<byte[], byte[]> secondTable = rdbStore
+ .getTable(families.get(2))) {
rdbStore.move(key, nextValue, firstTable, secondTable);
byte[] newvalue = secondTable.get(key);
// Make sure we have value in the second table
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java
index ed5fcb9..4dbf792 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java
@@ -95,7 +95,7 @@ public class TestRDBTableStore {
@Test
public void putGetAndEmpty() throws Exception {
- try (Table testTable = rdbStore.getTable("First")) {
+ try (Table<byte[], byte[]> testTable = rdbStore.getTable("First")) {
byte[] key =
RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8);
byte[] value =
@@ -209,7 +209,7 @@ public class TestRDBTableStore {
testTable.put(key, value);
}
int localCount = 0;
- try (TableIterator<Table.KeyValue> iter = testTable.iterator()) {
+ try (TableIterator<byte[], Table.KeyValue> iter = testTable.iterator()) {
while (iter.hasNext()) {
Table.KeyValue keyValue = iter.next();
localCount++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestTypedRDBTableStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestTypedRDBTableStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestTypedRDBTableStore.java
new file mode 100644
index 0000000..2f28497
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestTypedRDBTableStore.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.utils.db;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.utils.db.Table.KeyValue;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.Statistics;
+import org.rocksdb.StatsLevel;
+
+/**
+ * Tests for RocksDBTable Store.
+ */
+public class TestTypedRDBTableStore {
+ private static int count = 0;
+ private final List<String> families =
+ Arrays.asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
+ "First", "Second", "Third",
+ "Fourth", "Fifth",
+ "Sixth");
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+ private RDBStore rdbStore = null;
+ private DBOptions options = null;
+ private CodecRegistry codecRegistry;
+
+ @Before
+ public void setUp() throws Exception {
+ options = new DBOptions();
+ options.setCreateIfMissing(true);
+ options.setCreateMissingColumnFamilies(true);
+
+ Statistics statistics = new Statistics();
+ statistics.setStatsLevel(StatsLevel.ALL);
+ options = options.setStatistics(statistics);
+
+ Set<TableConfig> configSet = new HashSet<>();
+ for (String name : families) {
+ TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
+ configSet.add(newConfig);
+ }
+ rdbStore = new RDBStore(folder.newFolder(), options, configSet);
+
+ codecRegistry = new CodecRegistry();
+
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (rdbStore != null) {
+ rdbStore.close();
+ }
+ }
+
+ @Test
+ public void toIOException() {
+ }
+
+ @Test
+ public void putGetAndEmpty() throws Exception {
+ try (Table<String, String> testTable = createTypedTable(
+ "First")) {
+ String key =
+ RandomStringUtils.random(10);
+ String value = RandomStringUtils.random(10);
+ testTable.put(key, value);
+ Assert.assertFalse(testTable.isEmpty());
+ String readValue = testTable.get(key);
+ Assert.assertEquals(value, readValue);
+ }
+ try (Table secondTable = rdbStore.getTable("Second")) {
+ Assert.assertTrue(secondTable.isEmpty());
+ }
+ }
+
+ private Table<String, String> createTypedTable(String name)
+ throws IOException {
+ return new TypedTable<String, String>(
+ rdbStore.getTable(name),
+ codecRegistry,
+ String.class, String.class);
+ }
+
+ @Test
+ public void delete() throws Exception {
+ List<String> deletedKeys = new LinkedList<>();
+ List<String> validKeys = new LinkedList<>();
+ String value =
+ RandomStringUtils.random(10);
+ for (int x = 0; x < 100; x++) {
+ deletedKeys.add(
+ RandomStringUtils.random(10));
+ }
+
+ for (int x = 0; x < 100; x++) {
+ validKeys.add(
+ RandomStringUtils.random(10));
+ }
+
+ // Write all the keys and delete the keys scheduled for delete.
+ //Assert we find only expected keys in the Table.
+ try (Table<String, String> testTable = createTypedTable(
+ "Fourth")) {
+ for (int x = 0; x < deletedKeys.size(); x++) {
+ testTable.put(deletedKeys.get(x), value);
+ testTable.delete(deletedKeys.get(x));
+ }
+
+ for (int x = 0; x < validKeys.size(); x++) {
+ testTable.put(validKeys.get(x), value);
+ }
+
+ for (int x = 0; x < validKeys.size(); x++) {
+ Assert.assertNotNull(testTable.get(validKeys.get(0)));
+ }
+
+ for (int x = 0; x < deletedKeys.size(); x++) {
+ Assert.assertNull(testTable.get(deletedKeys.get(0)));
+ }
+ }
+ }
+
+ @Test
+ public void batchPut() throws Exception {
+
+ try (Table<String, String> testTable = createTypedTable(
+ "Fourth");
+ BatchOperation batch = rdbStore.initBatchOperation()) {
+ //given
+ String key =
+ RandomStringUtils.random(10);
+ String value =
+ RandomStringUtils.random(10);
+
+ //when
+ testTable.putWithBatch(batch, key, value);
+ rdbStore.commitBatchOperation(batch);
+
+ //then
+ Assert.assertNotNull(testTable.get(key));
+ }
+ }
+
+ @Test
+ public void batchDelete() throws Exception {
+ try (Table<String, String> testTable = createTypedTable(
+ "Fourth");
+ BatchOperation batch = rdbStore.initBatchOperation()) {
+
+ //given
+ String key =
+ RandomStringUtils.random(10);
+ String value =
+ RandomStringUtils.random(10);
+ testTable.put(key, value);
+
+ //when
+ testTable.deleteWithBatch(batch, key);
+ rdbStore.commitBatchOperation(batch);
+
+ //then
+ Assert.assertNull(testTable.get(key));
+ }
+ }
+
+ private static boolean consume(Table.KeyValue keyValue) {
+ count++;
+ Assert.assertNotNull(keyValue.getKey());
+ return true;
+ }
+
+ @Test
+ public void forEachAndIterator() throws Exception {
+ final int iterCount = 100;
+ try (Table<String, String> testTable = createTypedTable(
+ "Sixth")) {
+ for (int x = 0; x < iterCount; x++) {
+ String key =
+ RandomStringUtils.random(10);
+ String value =
+ RandomStringUtils.random(10);
+ testTable.put(key, value);
+ }
+ int localCount = 0;
+
+ try (TableIterator<String, ? extends KeyValue<String, String>> iter =
+ testTable.iterator()) {
+ while (iter.hasNext()) {
+ Table.KeyValue keyValue = iter.next();
+ localCount++;
+ }
+
+ Assert.assertEquals(iterCount, localCount);
+ iter.seekToFirst();
+ iter.forEachRemaining(TestTypedRDBTableStore::consume);
+ Assert.assertEquals(iterCount, count);
+
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 247911a..8989a68 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -203,50 +203,50 @@ public interface OMMetadataManager {
*
* @return UserTable.
*/
- Table getUserTable();
+ Table<byte[], byte[]> getUserTable();
/**
* Returns the Volume Table.
*
* @return VolumeTable.
*/
- Table getVolumeTable();
+ Table<byte[], byte[]> getVolumeTable();
/**
* Returns the BucketTable.
*
* @return BucketTable.
*/
- Table getBucketTable();
+ Table<byte[], byte[]> getBucketTable();
/**
* Returns the KeyTable.
*
* @return KeyTable.
*/
- Table getKeyTable();
+ Table<byte[], byte[]> getKeyTable();
/**
* Get Deleted Table.
*
* @return Deleted Table.
*/
- Table getDeletedTable();
+ Table<byte[], byte[]> getDeletedTable();
/**
* Gets the OpenKeyTable.
*
* @return Table.
*/
- Table getOpenKeyTable();
+ Table<byte[], byte[]> getOpenKeyTable();
/**
* Gets the S3Bucket to Ozone Volume/bucket mapping table.
*
* @return Table.
*/
- Table getS3Table();
+ Table<byte[], byte[]> getS3Table();
/**
* Returns number of rows in a table. This should not be used for very
* large tables.
@@ -254,5 +254,6 @@ public interface OMMetadataManager {
* @return long
* @throws IOException
*/
- long countRowsInTable(Table table) throws IOException;
+ <KEY, VALUE> long countRowsInTable(Table<KEY, VALUE> table)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
index 99c07ff..325ad2e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.db.Table;
+import org.apache.hadoop.utils.db.Table.KeyValue;
import org.apache.hadoop.utils.db.TableIterator;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -652,8 +653,7 @@ public class TestOzoneManager {
// Make sure the deleted key has been moved to the deleted table.
OMMetadataManager manager = cluster.getOzoneManager().
getMetadataManager();
-
- try(TableIterator<Table.KeyValue> iter =
+ try (TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> iter =
manager.getDeletedTable().iterator()) {
iter.seekToFirst();
Table.KeyValue kv = iter.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d15dc436/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index ad9dcc9..706a10f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -16,10 +16,16 @@
*/
package org.apache.hadoop.ozone.om;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -40,25 +46,20 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.DBStoreBuilder;
import org.apache.hadoop.utils.db.Table;
+import org.apache.hadoop.utils.db.Table.KeyValue;
import org.apache.hadoop.utils.db.TableIterator;
-import org.eclipse.jetty.util.StringUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import org.eclipse.jetty.util.StringUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Ozone metadata manager interface.
@@ -122,37 +123,37 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
}
@Override
- public Table getUserTable() {
+ public Table<byte[], byte[]> getUserTable() {
return userTable;
}
@Override
- public Table getVolumeTable() {
+ public Table<byte[], byte[]> getVolumeTable() {
return volumeTable;
}
@Override
- public Table getBucketTable() {
+ public Table<byte[], byte[]> getBucketTable() {
return bucketTable;
}
@Override
- public Table getKeyTable() {
+ public Table<byte[], byte[]> getKeyTable() {
return keyTable;
}
@Override
- public Table getDeletedTable() {
+ public Table<byte[], byte[]> getDeletedTable() {
return deletedTable;
}
@Override
- public Table getOpenKeyTable() {
+ public Table<byte[], byte[]> getOpenKeyTable() {
return openKeyTable;
}
@Override
- public Table getS3Table() {
+ public Table<byte[], byte[]> getS3Table() {
return s3Table;
}
@@ -349,8 +350,9 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
@Override
public boolean isVolumeEmpty(String volume) throws IOException {
byte[] volumePrefix = getVolumeKey(volume + OM_KEY_PREFIX);
- try (TableIterator<Table.KeyValue> bucketIter = bucketTable.iterator()) {
- Table.KeyValue kv = bucketIter.seek(volumePrefix);
+ try (TableIterator<byte[], Table.KeyValue> bucketIter = bucketTable
+ .iterator()) {
+ Table.KeyValue<byte[], byte[]> kv = bucketIter.seek(volumePrefix);
if (kv != null && startsWith(kv.getKey(), volumePrefix)) {
return false; // we found at least one bucket with this volume prefix.
}
@@ -370,8 +372,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
public boolean isBucketEmpty(String volume, String bucket)
throws IOException {
byte[] keyPrefix = getBucketKey(volume, bucket + OM_KEY_PREFIX);
- try (TableIterator<Table.KeyValue> keyIter = keyTable.iterator()) {
- Table.KeyValue kv = keyIter.seek(keyPrefix);
+ try (TableIterator<byte[], Table.KeyValue> keyIter = keyTable.iterator()) {
+ Table.KeyValue<byte[], byte[]> kv = keyIter.seek(keyPrefix);
if (kv != null && startsWith(kv.getKey(), keyPrefix)) {
return false; // we found at least one key with this vol/bucket prefix.
}
@@ -422,8 +424,9 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
seekPrefix = getVolumeKey(volumeName + OM_KEY_PREFIX);
}
int currentCount = 0;
- try (TableIterator<Table.KeyValue> bucketIter = bucketTable.iterator()) {
- Table.KeyValue kv = bucketIter.seek(startKey);
+ try (TableIterator<byte[], Table.KeyValue> bucketIter = bucketTable
+ .iterator()) {
+ Table.KeyValue<byte[], byte[]> kv = bucketIter.seek(startKey);
while (currentCount < maxNumOfBuckets && bucketIter.hasNext()) {
kv = bucketIter.next();
// Skip the Start Bucket if needed.
@@ -483,8 +486,10 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
seekPrefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX);
}
int currentCount = 0;
- try (TableIterator<Table.KeyValue> keyIter = getKeyTable().iterator()) {
- Table.KeyValue kv = keyIter.seek(seekKey);
+ try (TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> keyIter =
+ getKeyTable()
+ .iterator()) {
+ Table.KeyValue<byte[], byte[]> kv = keyIter.seek(seekKey);
while (currentCount < maxKeys && keyIter.hasNext()) {
kv = keyIter.next();
// Skip the Start key if needed.
@@ -578,10 +583,12 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
public List<BlockGroup> getPendingDeletionKeys(final int keyCount)
throws IOException {
List<BlockGroup> keyBlocksList = Lists.newArrayList();
- try (TableIterator<Table.KeyValue> keyIter = getDeletedTable().iterator()) {
+ try (TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> keyIter =
+ getDeletedTable()
+ .iterator()) {
int currentCount = 0;
while (keyIter.hasNext() && currentCount < keyCount) {
- Table.KeyValue kv = keyIter.next();
+ KeyValue<byte[], byte[]> kv = keyIter.next();
if (kv != null) {
OmKeyInfo info =
OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(kv.getValue()));
@@ -632,11 +639,12 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
}
@Override
- public long countRowsInTable(Table table) throws IOException {
+ public <KEY, VALUE> long countRowsInTable(Table<KEY, VALUE> table)
+ throws IOException {
long count = 0;
if (table != null) {
- try (TableIterator<Table.KeyValue> keyValueTableIterator =
- table.iterator()) {
+ try (TableIterator<KEY, ? extends KeyValue<KEY, VALUE>>
+ keyValueTableIterator = table.iterator()) {
while (keyValueTableIterator.hasNext()) {
keyValueTableIterator.next();
count++;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-887. Add DispatcherContext
info to Dispatcher from containerStateMachine. Contributed by Shashikant
Banerjee.
Posted by sh...@apache.org.
HDDS-887. Add DispatcherContext info to Dispatcher from containerStateMachine. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a3c7714
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a3c7714
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a3c7714
Branch: refs/heads/HDFS-12943
Commit: 5a3c7714c4d7822827ec365ea187fa8f43eb0e45
Parents: d15dc43
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Sun Dec 2 08:00:35 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Sun Dec 2 08:00:35 2018 +0530
----------------------------------------------------------------------
.../main/proto/DatanodeContainerProtocol.proto | 8 --
.../container/common/impl/HddsDispatcher.java | 8 +-
.../common/interfaces/ContainerDispatcher.java | 5 +-
.../container/common/interfaces/Handler.java | 4 +-
.../transport/server/GrpcXceiverService.java | 3 +-
.../transport/server/XceiverServerGrpc.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 120 +++++++----------
.../server/ratis/DispatcherContext.java | 133 +++++++++++++++++++
.../container/keyvalue/KeyValueHandler.java | 75 +++++++----
.../container/keyvalue/helpers/BlockUtils.java | 8 +-
.../keyvalue/helpers/SmallFileUtils.java | 10 +-
.../keyvalue/impl/ChunkManagerImpl.java | 4 +-
.../keyvalue/interfaces/ChunkManager.java | 5 +-
.../common/impl/TestHddsDispatcher.java | 14 +-
.../keyvalue/TestChunkManagerImpl.java | 17 +--
.../container/keyvalue/TestKeyValueHandler.java | 48 +++----
.../common/impl/TestContainerPersistence.java | 20 +--
.../transport/server/ratis/TestCSMMetrics.java | 3 +-
.../container/server/TestContainerServer.java | 6 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 16 +--
20 files changed, 321 insertions(+), 188 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
index 5237af8..661d910 100644
--- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto
@@ -373,17 +373,10 @@ enum ChecksumType {
MD5 = 5;
}
-enum Stage {
- WRITE_DATA = 1;
- COMMIT_DATA = 2;
- COMBINED = 3;
-}
-
message WriteChunkRequestProto {
required DatanodeBlockID blockID = 1;
required ChunkInfo chunkData = 2;
optional bytes data = 3;
- optional Stage stage = 4 [default = COMBINED];
}
message WriteChunkResponseProto {
@@ -392,7 +385,6 @@ message WriteChunkResponseProto {
message ReadChunkRequestProto {
required DatanodeBlockID blockID = 1;
required ChunkInfo chunkData = 2;
- optional bool readFromTmpFile = 3 [default = false];
}
message ReadChunkResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 352cc86..c5c51a3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -47,6 +47,8 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis
+ .DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -133,7 +135,7 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
@Override
public ContainerCommandResponseProto dispatch(
- ContainerCommandRequestProto msg) {
+ ContainerCommandRequestProto msg, DispatcherContext dispatcherContext) {
Preconditions.checkNotNull(msg);
LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
msg.getTraceID());
@@ -194,7 +196,7 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
audit(action, eventType, params, AuditEventStatus.FAILURE, ex);
return ContainerUtils.logAndReturnError(LOG, ex, msg);
}
- responseProto = handler.handle(msg, container);
+ responseProto = handler.handle(msg, container, dispatcherContext);
if (responseProto != null) {
metrics.incContainerOpsLatencies(cmdType, System.nanoTime() - startTime);
@@ -269,7 +271,7 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
// TODO: Assuming the container type to be KeyValueContainer for now.
// We need to get container type from the containerRequest.
Handler handler = getHandler(containerType);
- handler.handle(requestBuilder.build(), null);
+ handler.handle(requestBuilder.build(), null, null);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
index 7a22143..46a0b55 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandResponseProto;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
/**
* Dispatcher acts as the bridge between the transport layer and
@@ -37,9 +38,11 @@ public interface ContainerDispatcher {
/**
* Dispatches commands to container layer.
* @param msg - Command Request
+ * @param context - Context info related to ContainerStateMachine
* @return Command Response
*/
- ContainerCommandResponseProto dispatch(ContainerCommandRequestProto msg);
+ ContainerCommandResponseProto dispatch(ContainerCommandRequestProto msg,
+ DispatcherContext context);
/**
* Validates whether the container command should be executed on the pipeline
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index ad55618..9f520d5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerExcep
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
@@ -103,7 +104,8 @@ public abstract class Handler {
}
public abstract ContainerCommandResponseProto handle(
- ContainerCommandRequestProto msg, Container container);
+ ContainerCommandRequestProto msg, Container container,
+ DispatcherContext dispatcherContext);
/**
* Import container data from a raw input stream.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
index 5fc3661..37b7d5d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/GrpcXceiverService.java
@@ -53,7 +53,8 @@ public class GrpcXceiverService extends
@Override
public void onNext(ContainerCommandRequestProto request) {
try {
- ContainerCommandResponseProto resp = dispatcher.dispatch(request);
+ ContainerCommandResponseProto resp =
+ dispatcher.dispatch(request, null);
responseObserver.onNext(resp);
} catch (Throwable e) {
LOG.error("{} got exception when processing"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 4f7799d..4ed9653 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -130,7 +130,7 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
public void submitRequest(ContainerCommandRequestProto request,
HddsProtos.PipelineID pipelineID) throws IOException {
ContainerProtos.ContainerCommandResponseProto response =
- storageContainer.dispatch(request);
+ storageContainer.dispatch(request, null);
if (response.getResult() != ContainerProtos.Result.SUCCESS) {
throw new StorageContainerException(response.getMessage(),
response.getResult());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 2e0c70e..b693e9e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -22,8 +22,6 @@ import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
@@ -35,7 +33,6 @@ import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo;
import org.apache.ratis.thirdparty.com.google.protobuf
.InvalidProtocolBufferException;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Stage;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -237,7 +234,6 @@ public class ContainerStateMachine extends BaseStateMachine {
final WriteChunkRequestProto dataWriteChunkProto =
WriteChunkRequestProto
.newBuilder(write)
- .setStage(Stage.WRITE_DATA)
.build();
ContainerCommandRequestProto dataContainerCommandProto =
ContainerCommandRequestProto
@@ -252,7 +248,6 @@ public class ContainerStateMachine extends BaseStateMachine {
.setChunkData(write.getChunkData())
// skipping the data field as it is
// already set in statemachine data proto
- .setStage(Stage.COMMIT_DATA)
.build();
ContainerCommandRequestProto commitContainerCommandProto =
ContainerCommandRequestProto
@@ -292,15 +287,18 @@ public class ContainerStateMachine extends BaseStateMachine {
}
private ContainerCommandResponseProto dispatchCommand(
- ContainerCommandRequestProto requestProto) {
+ ContainerCommandRequestProto requestProto,
+ DispatcherContext context) {
LOG.trace("dispatch {}", requestProto);
- ContainerCommandResponseProto response = dispatcher.dispatch(requestProto);
+ ContainerCommandResponseProto response =
+ dispatcher.dispatch(requestProto, context);
LOG.trace("response {}", response);
return response;
}
- private Message runCommand(ContainerCommandRequestProto requestProto) {
- return dispatchCommand(requestProto)::toByteString;
+ private Message runCommand(ContainerCommandRequestProto requestProto,
+ DispatcherContext context) {
+ return dispatchCommand(requestProto, context)::toByteString;
}
private ExecutorService getCommandExecutor(
@@ -310,7 +308,7 @@ public class ContainerStateMachine extends BaseStateMachine {
}
private CompletableFuture<Message> handleWriteChunk(
- ContainerCommandRequestProto requestProto, long entryIndex) {
+ ContainerCommandRequestProto requestProto, long entryIndex, long term) {
final WriteChunkRequestProto write = requestProto.getWriteChunk();
RaftServer server = ratisServer.getServer();
Preconditions.checkState(server instanceof RaftServerProxy);
@@ -321,8 +319,14 @@ public class ContainerStateMachine extends BaseStateMachine {
} catch (IOException ioe) {
return completeExceptionally(ioe);
}
+ DispatcherContext context =
+ new DispatcherContext.Builder()
+ .setTerm(term)
+ .setLogIndex(entryIndex)
+ .setStage(DispatcherContext.WriteChunkStage.WRITE_DATA)
+ .build();
CompletableFuture<Message> writeChunkFuture = CompletableFuture
- .supplyAsync(() -> runCommand(requestProto), chunkExecutor);
+ .supplyAsync(() -> runCommand(requestProto, context), chunkExecutor);
writeChunkFutureMap.put(entryIndex, writeChunkFuture);
LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID()
+ " logIndex " + entryIndex + " chunkName " + write.getChunkData()
@@ -355,7 +359,8 @@ public class ContainerStateMachine extends BaseStateMachine {
// CreateContainer will happen as a part of writeChunk only.
switch (cmdType) {
case WriteChunk:
- return handleWriteChunk(requestProto, entry.getIndex());
+ return handleWriteChunk(requestProto, entry.getIndex(),
+ entry.getTerm());
default:
throw new IllegalStateException("Cmd Type:" + cmdType
+ " should not have state machine data");
@@ -372,39 +377,36 @@ public class ContainerStateMachine extends BaseStateMachine {
metrics.incNumReadStateMachineOps();
final ContainerCommandRequestProto requestProto =
getRequestProto(request.getContent());
- return CompletableFuture.completedFuture(runCommand(requestProto));
+ return CompletableFuture.completedFuture(runCommand(requestProto, null));
} catch (IOException e) {
metrics.incNumReadStateMachineFails();
return completeExceptionally(e);
}
}
- private ByteString readStateMachineData(ContainerCommandRequestProto
- requestProto) {
+ private ByteString readStateMachineData(
+ ContainerCommandRequestProto requestProto, long term, long index) {
WriteChunkRequestProto writeChunkRequestProto =
requestProto.getWriteChunk();
- // Assert that store log entry is for COMMIT_DATA, the WRITE_DATA is
- // written through writeStateMachineData.
- Preconditions
- .checkArgument(writeChunkRequestProto.getStage() == Stage.COMMIT_DATA);
-
// prepare the chunk to be read
ReadChunkRequestProto.Builder readChunkRequestProto =
ReadChunkRequestProto.newBuilder()
.setBlockID(writeChunkRequestProto.getBlockID())
- .setChunkData(writeChunkRequestProto.getChunkData())
- // set readFromTempFile to true in case, the chunkFile does
- // not exist as applyTransaction is not executed for this entry yet.
- .setReadFromTmpFile(true);
+ .setChunkData(writeChunkRequestProto.getChunkData());
ContainerCommandRequestProto dataContainerCommandProto =
ContainerCommandRequestProto.newBuilder(requestProto)
.setCmdType(Type.ReadChunk)
.setReadChunk(readChunkRequestProto)
.build();
-
+ DispatcherContext context =
+ new DispatcherContext.Builder()
+ .setTerm(term)
+ .setLogIndex(index)
+ .setReadFromTmpFile(true)
+ .build();
// read the chunk
ContainerCommandResponseProto response =
- dispatchCommand(dataContainerCommandProto);
+ dispatchCommand(dataContainerCommandProto, context);
ReadChunkResponseProto responseProto = response.getReadChunk();
ByteString data = responseProto.getData();
@@ -416,14 +418,14 @@ public class ContainerStateMachine extends BaseStateMachine {
/**
* Reads the Entry from the Cache or loads it back by reading from disk.
*/
- private ByteString getCachedStateMachineData(Long logIndex,
+ private ByteString getCachedStateMachineData(Long logIndex, long term,
ContainerCommandRequestProto requestProto) throws ExecutionException {
try {
return reconstructWriteChunkRequest(
stateMachineDataCache.get(logIndex, new Callable<ByteString>() {
@Override
public ByteString call() throws Exception {
- return readStateMachineData(requestProto);
+ return readStateMachineData(requestProto, term, logIndex);
}
}), requestProto);
} catch (ExecutionException e) {
@@ -439,7 +441,7 @@ public class ContainerStateMachine extends BaseStateMachine {
final WriteChunkRequestProto.Builder dataWriteChunkProto =
WriteChunkRequestProto.newBuilder(writeChunkRequestProto)
// adding the state machine data
- .setData(data).setStage(Stage.WRITE_DATA);
+ .setData(data);
ContainerCommandRequestProto.Builder newStateMachineProto =
ContainerCommandRequestProto.newBuilder(requestProto)
@@ -486,7 +488,8 @@ public class ContainerStateMachine extends BaseStateMachine {
CompletableFuture<ByteString> future = new CompletableFuture<>();
return future.supplyAsync(() -> {
try {
- return getCachedStateMachineData(entry.getIndex(), requestProto);
+ return getCachedStateMachineData(entry.getIndex(), entry.getTerm(),
+ requestProto);
} catch (ExecutionException e) {
future.completeExceptionally(e);
return null;
@@ -524,6 +527,10 @@ public class ContainerStateMachine extends BaseStateMachine {
@Override
public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
long index = trx.getLogEntry().getIndex();
+ DispatcherContext.Builder builder =
+ new DispatcherContext.Builder()
+ .setTerm(trx.getLogEntry().getTerm())
+ .setLogIndex(index);
// ApplyTransaction call can come with an entryIndex much greater than
// lastIndex updated because in between entries in the raft log can be
@@ -539,51 +546,16 @@ public class ContainerStateMachine extends BaseStateMachine {
getRequestProto(trx.getStateMachineLogEntry().getLogData());
Type cmdType = requestProto.getCmdType();
CompletableFuture<Message> future;
- if (cmdType == Type.PutBlock || cmdType == Type.PutSmallFile) {
- BlockData blockData;
- ContainerProtos.BlockData blockDataProto = cmdType == Type.PutBlock ?
- requestProto.getPutBlock().getBlockData() :
- requestProto.getPutSmallFile().getBlock().getBlockData();
-
- // set the blockCommitSequenceId
- try {
- blockData = BlockData.getFromProtoBuf(blockDataProto);
- } catch (IOException ioe) {
- LOG.error("unable to retrieve blockData info for Block {}",
- blockDataProto.getBlockID());
- return completeExceptionally(ioe);
- }
- blockData.setBlockCommitSequenceId(index);
- final ContainerProtos.PutBlockRequestProto putBlockRequestProto =
- ContainerProtos.PutBlockRequestProto
- .newBuilder(requestProto.getPutBlock())
- .setBlockData(blockData.getProtoBufMessage()).build();
- ContainerCommandRequestProto containerCommandRequestProto;
- if (cmdType == Type.PutSmallFile) {
- ContainerProtos.PutSmallFileRequestProto smallFileRequestProto =
- ContainerProtos.PutSmallFileRequestProto
- .newBuilder(requestProto.getPutSmallFile())
- .setBlock(putBlockRequestProto).build();
- containerCommandRequestProto =
- ContainerCommandRequestProto.newBuilder(requestProto)
- .setPutSmallFile(smallFileRequestProto).build();
- } else {
- containerCommandRequestProto =
- ContainerCommandRequestProto.newBuilder(requestProto)
- .setPutBlock(putBlockRequestProto).build();
- }
- future = CompletableFuture
- .supplyAsync(() -> runCommand(containerCommandRequestProto),
- getCommandExecutor(requestProto));
- } else {
- // Make sure that in write chunk, the user data is not set
- if (cmdType == Type.WriteChunk) {
- Preconditions.checkArgument(requestProto
- .getWriteChunk().getData().isEmpty());
- }
- future = CompletableFuture.supplyAsync(() -> runCommand(requestProto),
- getCommandExecutor(requestProto));
+ // Make sure that in write chunk, the user data is not set
+ if (cmdType == Type.WriteChunk) {
+ Preconditions
+ .checkArgument(requestProto.getWriteChunk().getData().isEmpty());
+ builder
+ .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA);
}
+ future = CompletableFuture
+ .supplyAsync(() -> runCommand(requestProto, builder.build()),
+ getCommandExecutor(requestProto));
lastIndex = index;
future.thenAccept(m -> {
final Long previous =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java
new file mode 100644
index 0000000..a46e6b8
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.transport.server.ratis;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * DispatcherContext class holds transport protocol specfic context info
+ * required for execution of container commands over the container dispatcher.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DispatcherContext {
+ /**
+ * Determines which stage of writeChunk a write chunk request is for.
+ */
+ public enum WriteChunkStage {
+ WRITE_DATA, COMMIT_DATA, COMBINED
+ }
+
+ // whether the chunk data needs to be written or committed or both
+ private final WriteChunkStage stage;
+ // indicates whether the read from tmp chunk files is allowed
+ private final boolean readFromTmpFile;
+ // which term the request is being served in Ratis
+ private final long term;
+ // the log index in Ratis log to which the request belongs to
+ private final long logIndex;
+
+ private DispatcherContext(long term, long index, WriteChunkStage stage,
+ boolean readFromTmpFile) {
+ this.term = term;
+ this.logIndex = index;
+ this.stage = stage;
+ this.readFromTmpFile = readFromTmpFile;
+ }
+
+ public long getLogIndex() {
+ return logIndex;
+ }
+
+ public boolean isReadFromTmpFile() {
+ return readFromTmpFile;
+ }
+
+ public long getTerm() {
+ return term;
+ }
+
+ public WriteChunkStage getStage() {
+ return stage;
+ }
+
+ /**
+ * Builder class for building DispatcherContext.
+ */
+ public static final class Builder {
+ private WriteChunkStage stage = WriteChunkStage.COMBINED;
+ private boolean readFromTmpFile = false;
+ private long term;
+ private long logIndex;
+
+ /**
+ * Sets the WriteChunkStage.
+ *
+ * @param stage WriteChunk Stage
+ * @return DispatcherContext.Builder
+ */
+ public Builder setStage(WriteChunkStage stage) {
+ this.stage = stage;
+ return this;
+ }
+
+ /**
+ * Sets the flag for reading from tmp chunk files.
+ *
+ * @param readFromTmpFile whether to read from tmp chunk file or not
+ * @return DispatcherContext.Builder
+ */
+ public Builder setReadFromTmpFile(boolean readFromTmpFile) {
+ this.readFromTmpFile = readFromTmpFile;
+ return this;
+ }
+
+ /**
+ * Sets the current term for the container request from Ratis.
+ *
+ * @param term current term
+ * @return DispatcherContext.Builder
+ */
+ public Builder setTerm(long term) {
+ this.term = term;
+ return this;
+ }
+
+ /**
+ * Sets the logIndex for the container request from Ratis.
+ *
+ * @param logIndex log index
+ * @return DispatcherContext.Builder
+ */
+ public Builder setLogIndex(long logIndex) {
+ this.logIndex = logIndex;
+ return this;
+ }
+
+ /**
+ * Builds and returns DatanodeDetails instance.
+ *
+ * @return DispatcherContext
+ */
+ public DispatcherContext build() {
+ return new DispatcherContext(term, logIndex, stage, readFromTmpFile);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index a16129e..b4cfcd0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -58,6 +58,10 @@ import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis
+ .DispatcherContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis
+ .DispatcherContext.WriteChunkStage;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume
.RoundRobinVolumeChoosingPolicy;
@@ -81,8 +85,6 @@ import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import static org.apache.hadoop.hdds.HddsConfigKeys
.HDDS_DATANODE_VOLUME_CHOOSING_POLICY;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.*;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .Stage;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys
@@ -146,7 +148,8 @@ public class KeyValueHandler extends Handler {
@Override
public ContainerCommandResponseProto handle(
- ContainerCommandRequestProto request, Container container) {
+ ContainerCommandRequestProto request, Container container,
+ DispatcherContext dispatcherContext) {
Type cmdType = request.getCmdType();
KeyValueContainer kvContainer = (KeyValueContainer) container;
@@ -164,7 +167,7 @@ public class KeyValueHandler extends Handler {
case CloseContainer:
return handleCloseContainer(request, kvContainer);
case PutBlock:
- return handlePutBlock(request, kvContainer);
+ return handlePutBlock(request, kvContainer, dispatcherContext);
case GetBlock:
return handleGetBlock(request, kvContainer);
case DeleteBlock:
@@ -172,17 +175,17 @@ public class KeyValueHandler extends Handler {
case ListBlock:
return handleUnsupportedOp(request);
case ReadChunk:
- return handleReadChunk(request, kvContainer);
+ return handleReadChunk(request, kvContainer, dispatcherContext);
case DeleteChunk:
return handleDeleteChunk(request, kvContainer);
case WriteChunk:
- return handleWriteChunk(request, kvContainer);
+ return handleWriteChunk(request, kvContainer, dispatcherContext);
case ListChunk:
return handleUnsupportedOp(request);
case CompactChunk:
return handleUnsupportedOp(request);
case PutSmallFile:
- return handlePutSmallFile(request, kvContainer);
+ return handlePutSmallFile(request, kvContainer, dispatcherContext);
case GetSmallFile:
return handleGetSmallFile(request, kvContainer);
case GetCommittedBlockLength:
@@ -392,7 +395,8 @@ public class KeyValueHandler extends Handler {
* Handle Put Block operation. Calls BlockManager to process the request.
*/
ContainerCommandResponseProto handlePutBlock(
- ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer,
+ DispatcherContext dispatcherContext) {
long blockLength;
if (!request.hasPutBlock()) {
@@ -401,14 +405,18 @@ public class KeyValueHandler extends Handler {
return ContainerUtils.malformedRequest(request);
}
+ BlockData blockData;
try {
checkContainerOpen(kvContainer);
- BlockData blockData = BlockData.getFromProtoBuf(
+ blockData = BlockData.getFromProtoBuf(
request.getPutBlock().getBlockData());
Preconditions.checkNotNull(blockData);
+ long bcsId =
+ dispatcherContext == null ? 0 : dispatcherContext.getLogIndex();
+ blockData.setBlockCommitSequenceId(bcsId);
long numBytes = blockData.getProtoBufMessage().toByteArray().length;
- blockLength = blockManager.putBlock(kvContainer, blockData);
+ blockManager.putBlock(kvContainer, blockData);
metrics.incContainerBytesStats(Type.PutBlock, numBytes);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -418,7 +426,7 @@ public class KeyValueHandler extends Handler {
request);
}
- return BlockUtils.putBlockResponseSuccess(request, blockLength);
+ return BlockUtils.putBlockResponseSuccess(request, blockData);
}
/**
@@ -514,7 +522,8 @@ public class KeyValueHandler extends Handler {
* Handle Read Chunk operation. Calls ChunkManager to process the request.
*/
ContainerCommandResponseProto handleReadChunk(
- ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer,
+ DispatcherContext dispatcherContext) {
if (!request.hasReadChunk()) {
LOG.debug("Malformed Read Chunk request. trace ID: {}",
@@ -531,8 +540,10 @@ public class KeyValueHandler extends Handler {
.getChunkData());
Preconditions.checkNotNull(chunkInfo);
- data = chunkManager.readChunk(kvContainer, blockID, chunkInfo,
- request.getReadChunk().getReadFromTmpFile());
+ boolean isReadFromTmpFile = dispatcherContext == null ? false :
+ dispatcherContext.isReadFromTmpFile();
+ data = chunkManager
+ .readChunk(kvContainer, blockID, chunkInfo, isReadFromTmpFile);
metrics.incContainerBytesStats(Type.ReadChunk, data.length);
} catch (StorageContainerException ex) {
return ContainerUtils.logAndReturnError(LOG, ex, request);
@@ -583,7 +594,8 @@ public class KeyValueHandler extends Handler {
* Handle Write Chunk operation. Calls ChunkManager to process the request.
*/
ContainerCommandResponseProto handleWriteChunk(
- ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer,
+ DispatcherContext dispatcherContext) {
if (!request.hasWriteChunk()) {
LOG.debug("Malformed Write Chunk request. trace ID: {}",
@@ -602,17 +614,19 @@ public class KeyValueHandler extends Handler {
Preconditions.checkNotNull(chunkInfo);
ByteBuffer data = null;
- if (request.getWriteChunk().getStage() == Stage.WRITE_DATA ||
- request.getWriteChunk().getStage() == Stage.COMBINED) {
+ WriteChunkStage stage =
+ dispatcherContext == null ? WriteChunkStage.COMBINED :
+ dispatcherContext.getStage();
+ if (stage == WriteChunkStage.WRITE_DATA ||
+ stage == WriteChunkStage.COMBINED) {
data = request.getWriteChunk().getData().asReadOnlyByteBuffer();
}
- chunkManager.writeChunk(kvContainer, blockID, chunkInfo, data,
- request.getWriteChunk().getStage());
+ chunkManager.writeChunk(kvContainer, blockID, chunkInfo, data, stage);
// We should increment stats after writeChunk
- if (request.getWriteChunk().getStage() == Stage.WRITE_DATA ||
- request.getWriteChunk().getStage() == Stage.COMBINED) {
+ if (stage == WriteChunkStage.WRITE_DATA||
+ stage == WriteChunkStage.COMBINED) {
metrics.incContainerBytesStats(Type.WriteChunk, request.getWriteChunk()
.getChunkData().getLen());
}
@@ -633,7 +647,8 @@ public class KeyValueHandler extends Handler {
* request.
*/
ContainerCommandResponseProto handlePutSmallFile(
- ContainerCommandRequestProto request, KeyValueContainer kvContainer) {
+ ContainerCommandRequestProto request, KeyValueContainer kvContainer,
+ DispatcherContext dispatcherContext) {
if (!request.hasPutSmallFile()) {
LOG.debug("Malformed Put Small File request. trace ID: {}",
@@ -642,13 +657,14 @@ public class KeyValueHandler extends Handler {
}
PutSmallFileRequestProto putSmallFileReq =
request.getPutSmallFile();
+ BlockData blockData;
try {
checkContainerOpen(kvContainer);
BlockID blockID = BlockID.getFromProtobuf(putSmallFileReq.getBlock()
.getBlockData().getBlockID());
- BlockData blockData = BlockData.getFromProtoBuf(
+ blockData = BlockData.getFromProtoBuf(
putSmallFileReq.getBlock().getBlockData());
Preconditions.checkNotNull(blockData);
@@ -656,15 +672,20 @@ public class KeyValueHandler extends Handler {
putSmallFileReq.getChunkInfo());
Preconditions.checkNotNull(chunkInfo);
ByteBuffer data = putSmallFileReq.getData().asReadOnlyByteBuffer();
+ WriteChunkStage stage =
+ dispatcherContext == null ? WriteChunkStage.COMBINED :
+ dispatcherContext.getStage();
// chunks will be committed as a part of handling putSmallFile
// here. There is no need to maintain this info in openContainerBlockMap.
- chunkManager.writeChunk(
- kvContainer, blockID, chunkInfo, data, Stage.COMBINED);
+ chunkManager.writeChunk(kvContainer, blockID, chunkInfo, data, stage);
List<ContainerProtos.ChunkInfo> chunks = new LinkedList<>();
chunks.add(chunkInfo.getProtoBufMessage());
blockData.setChunks(chunks);
- // TODO: add bcsId as a part of putSmallFile transaction
+ long bcsId =
+ dispatcherContext == null ? 0 : dispatcherContext.getLogIndex();
+ blockData.setBlockCommitSequenceId(bcsId);
+
blockManager.putBlock(kvContainer, blockData);
metrics.incContainerBytesStats(Type.PutSmallFile, data.capacity());
@@ -676,7 +697,7 @@ public class KeyValueHandler extends Handler {
PUT_SMALL_FILE_ERROR), request);
}
- return SmallFileUtils.getPutFileResponseSuccess(request);
+ return SmallFileUtils.getPutFileResponseSuccess(request, blockData);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
index 667e66d..200e8ea 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -133,12 +133,12 @@ public final class BlockUtils {
* @return Response.
*/
public static ContainerCommandResponseProto putBlockResponseSuccess(
- ContainerCommandRequestProto msg, long blockLength) {
- ContainerProtos.BlockData blockData = msg.getPutBlock().getBlockData();
+ ContainerCommandRequestProto msg, BlockData blockData) {
+ ContainerProtos.BlockData blockDataProto = blockData.getProtoBufMessage();
GetCommittedBlockLengthResponseProto.Builder
committedBlockLengthResponseBuilder =
- getCommittedBlockLengthResponseBuilder(blockLength,
- blockData.getBlockID());
+ getCommittedBlockLengthResponseBuilder(blockData.getSize(),
+ blockDataProto.getBlockID());
PutBlockResponseProto.Builder putKeyResponse =
PutBlockResponseProto.newBuilder();
putKeyResponse
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
index e91c8a6..ba2b02c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.container.keyvalue.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
@@ -44,15 +45,14 @@ public final class SmallFileUtils {
* @return - ContainerCommandResponseProto
*/
public static ContainerCommandResponseProto getPutFileResponseSuccess(
- ContainerCommandRequestProto msg) {
+ ContainerCommandRequestProto msg, BlockData blockData) {
ContainerProtos.PutSmallFileResponseProto.Builder getResponse =
ContainerProtos.PutSmallFileResponseProto.newBuilder();
- ContainerProtos.BlockData blockData =
- msg.getPutSmallFile().getBlock().getBlockData();
+ ContainerProtos.BlockData blockDataProto = blockData.getProtoBufMessage();
ContainerProtos.GetCommittedBlockLengthResponseProto.Builder
committedBlockLengthResponseBuilder = BlockUtils
- .getCommittedBlockLengthResponseBuilder(blockData.getSize(),
- blockData.getBlockID());
+ .getCommittedBlockLengthResponseBuilder(blockDataProto.getSize(),
+ blockDataProto.getBlockID());
getResponse.setCommittedBlockLength(committedBlockLengthResponseBuilder);
ContainerCommandResponseProto.Builder builder =
ContainerUtils.getSuccessResponseBuilder(msg);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index e41346c..38430af 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -21,10 +21,10 @@ package org.apache.hadoop.ozone.container.keyvalue.impl;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
@@ -66,7 +66,7 @@ public class ChunkManagerImpl implements ChunkManager {
* @throws StorageContainerException
*/
public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
- ByteBuffer data, ContainerProtos.Stage stage)
+ ByteBuffer data, DispatcherContext.WriteChunkStage stage)
throws StorageContainerException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
index f90346e..4282e46 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/ChunkManager.java
@@ -19,10 +19,11 @@ package org.apache.hadoop.ozone.container.keyvalue.interfaces;
*/
import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
+
import java.nio.ByteBuffer;
/**
@@ -42,7 +43,7 @@ public interface ChunkManager {
* @throws StorageContainerException
*/
void writeChunk(Container container, BlockID blockID, ChunkInfo info,
- ByteBuffer data, ContainerProtos.Stage stage)
+ ByteBuffer data, DispatcherContext.WriteChunkStage stage)
throws StorageContainerException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index 9940d4d..933ed70 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -99,16 +99,16 @@ public class TestHddsDispatcher {
HddsDispatcher hddsDispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, context, metrics);
hddsDispatcher.setScmId(scmId.toString());
- ContainerCommandResponseProto responseOne = hddsDispatcher.dispatch(
- getWriteChunkRequest(dd.getUuidString(), 1L, 1L));
+ ContainerCommandResponseProto responseOne = hddsDispatcher
+ .dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 1L), null);
Assert.assertEquals(ContainerProtos.Result.SUCCESS,
responseOne.getResult());
verify(context, times(0))
.addContainerActionIfAbsent(Mockito.any(ContainerAction.class));
containerData.setBytesUsed(Double.valueOf(
StorageUnit.MB.toBytes(950)).longValue());
- ContainerCommandResponseProto responseTwo = hddsDispatcher.dispatch(
- getWriteChunkRequest(dd.getUuidString(), 1L, 2L));
+ ContainerCommandResponseProto responseTwo = hddsDispatcher
+ .dispatch(getWriteChunkRequest(dd.getUuidString(), 1L, 2L), null);
Assert.assertEquals(ContainerProtos.Result.SUCCESS,
responseTwo.getResult());
verify(context, times(1))
@@ -150,16 +150,16 @@ public class TestHddsDispatcher {
getWriteChunkRequest(dd.getUuidString(), 1L, 1L);
// send read chunk request and make sure container does not exist
ContainerCommandResponseProto response =
- hddsDispatcher.dispatch(getReadChunkRequest(writeChunkRequest));
+ hddsDispatcher.dispatch(getReadChunkRequest(writeChunkRequest), null);
Assert.assertEquals(response.getResult(),
ContainerProtos.Result.CONTAINER_NOT_FOUND);
// send write chunk request without sending create container
- response = hddsDispatcher.dispatch(writeChunkRequest);
+ response = hddsDispatcher.dispatch(writeChunkRequest, null);
// container should be created as part of write chunk request
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
// send read chunk request to read the chunk written above
response =
- hddsDispatcher.dispatch(getReadChunkRequest(writeChunkRequest));
+ hddsDispatcher.dispatch(getReadChunkRequest(writeChunkRequest), null);
Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult());
Assert.assertEquals(response.getReadChunk().getData(),
writeChunkRequest.getWriteChunk().getData());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
index ef77204..f17288f 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext.WriteChunkStage;
import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
@@ -114,7 +115,7 @@ public class TestChunkManagerImpl {
// As no chunks are written to the volume writeBytes should be 0
checkWriteIOStats(0, 0);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.WRITE_DATA);
+ ByteBuffer.wrap(data), WriteChunkStage.WRITE_DATA);
// Now a chunk file is being written with Stage WRITE_DATA, so it should
// create a temporary chunk file.
assertTrue(chunksPath.listFiles().length == 1);
@@ -131,7 +132,7 @@ public class TestChunkManagerImpl {
checkWriteIOStats(data.length, 1);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.COMMIT_DATA);
+ ByteBuffer.wrap(data), WriteChunkStage.COMMIT_DATA);
checkWriteIOStats(data.length, 1);
@@ -151,7 +152,7 @@ public class TestChunkManagerImpl {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, randomLength);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.WRITE_DATA);
+ ByteBuffer.wrap(data), WriteChunkStage.WRITE_DATA);
fail("testWriteChunkIncorrectLength failed");
} catch (StorageContainerException ex) {
// As we got an exception, writeBytes should be 0.
@@ -172,7 +173,7 @@ public class TestChunkManagerImpl {
assertTrue(chunksPath.listFiles().length == 0);
checkWriteIOStats(0, 0);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
+ ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
// Now a chunk file is being written with Stage COMBINED_DATA, so it should
// create a chunk file.
assertTrue(chunksPath.listFiles().length == 1);
@@ -185,7 +186,7 @@ public class TestChunkManagerImpl {
public void testReadChunk() throws Exception {
checkWriteIOStats(0, 0);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
+ ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
checkWriteIOStats(data.length, 1);
checkReadIOStats(0, 0);
byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID,
@@ -199,7 +200,7 @@ public class TestChunkManagerImpl {
public void testDeleteChunk() throws Exception {
File chunksPath = new File(keyValueContainerData.getChunksPath());
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
+ ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
assertTrue(chunksPath.listFiles().length == 1);
chunkManager.deleteChunk(keyValueContainer, blockID, chunkInfo);
assertTrue(chunksPath.listFiles().length == 0);
@@ -209,7 +210,7 @@ public class TestChunkManagerImpl {
public void testDeleteChunkUnsupportedRequest() throws Exception {
try {
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
+ ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
long randomLength = 200L;
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), 0), 0, randomLength);
@@ -241,7 +242,7 @@ public class TestChunkManagerImpl {
chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID
.getLocalID(), i), 0, data.length);
chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo,
- ByteBuffer.wrap(data), ContainerProtos.Stage.COMBINED);
+ ByteBuffer.wrap(data), WriteChunkStage.COMBINED);
}
checkWriteIOStats(data.length*100, 100);
assertTrue(hddsVolume.getVolumeIOStats().getWriteTime() > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 29d74c2..a2e7f50 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
@@ -84,10 +85,10 @@ public class TestKeyValueHandler {
handler = Mockito.mock(KeyValueHandler.class);
dispatcher = Mockito.mock(HddsDispatcher.class);
Mockito.when(dispatcher.getHandler(any())).thenReturn(handler);
- Mockito.when(dispatcher.dispatch(any())).thenCallRealMethod();
+ Mockito.when(dispatcher.dispatch(any(), any())).thenCallRealMethod();
Mockito.when(dispatcher.getContainer(anyLong())).thenReturn(
Mockito.mock(KeyValueContainer.class));
- Mockito.when(handler.handle(any(), any())).thenCallRealMethod();
+ Mockito.when(handler.handle(any(), any(), any())).thenCallRealMethod();
doCallRealMethod().when(dispatcher).setMetricsForTesting(any());
dispatcher.setMetricsForTesting(Mockito.mock(ContainerMetrics.class));
Mockito.when(dispatcher.buildAuditMessageForFailure(any(), any(), any()))
@@ -111,112 +112,113 @@ public class TestKeyValueHandler {
.setCreateContainer(ContainerProtos.CreateContainerRequestProto
.getDefaultInstance())
.build();
- dispatcher.dispatch(createContainerRequest);
+ DispatcherContext context = new DispatcherContext.Builder().build();
+ dispatcher.dispatch(createContainerRequest, context);
Mockito.verify(handler, times(1)).handleCreateContainer(
any(ContainerCommandRequestProto.class), any());
// Test Read Container Request handling
ContainerCommandRequestProto readContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ReadContainer);
- dispatcher.dispatch(readContainerRequest);
+ dispatcher.dispatch(readContainerRequest, context);
Mockito.verify(handler, times(1)).handleReadContainer(
any(ContainerCommandRequestProto.class), any());
// Test Update Container Request handling
ContainerCommandRequestProto updateContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.UpdateContainer);
- dispatcher.dispatch(updateContainerRequest);
+ dispatcher.dispatch(updateContainerRequest, context);
Mockito.verify(handler, times(1)).handleUpdateContainer(
any(ContainerCommandRequestProto.class), any());
// Test Delete Container Request handling
ContainerCommandRequestProto deleteContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.DeleteContainer);
- dispatcher.dispatch(deleteContainerRequest);
+ dispatcher.dispatch(deleteContainerRequest, null);
Mockito.verify(handler, times(1)).handleDeleteContainer(
any(ContainerCommandRequestProto.class), any());
// Test List Container Request handling
ContainerCommandRequestProto listContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ListContainer);
- dispatcher.dispatch(listContainerRequest);
+ dispatcher.dispatch(listContainerRequest, context);
Mockito.verify(handler, times(1)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Close Container Request handling
ContainerCommandRequestProto closeContainerRequest =
getDummyCommandRequestProto(ContainerProtos.Type.CloseContainer);
- dispatcher.dispatch(closeContainerRequest);
+ dispatcher.dispatch(closeContainerRequest, context);
Mockito.verify(handler, times(1)).handleCloseContainer(
any(ContainerCommandRequestProto.class), any());
// Test Put Block Request handling
ContainerCommandRequestProto putBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.PutBlock);
- dispatcher.dispatch(putBlockRequest);
+ dispatcher.dispatch(putBlockRequest, context);
Mockito.verify(handler, times(1)).handlePutBlock(
- any(ContainerCommandRequestProto.class), any());
+ any(ContainerCommandRequestProto.class), any(), any());
// Test Get Block Request handling
ContainerCommandRequestProto getBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.GetBlock);
- dispatcher.dispatch(getBlockRequest);
+ dispatcher.dispatch(getBlockRequest, context);
Mockito.verify(handler, times(1)).handleGetBlock(
any(ContainerCommandRequestProto.class), any());
// Test Delete Block Request handling
ContainerCommandRequestProto deleteBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.DeleteBlock);
- dispatcher.dispatch(deleteBlockRequest);
+ dispatcher.dispatch(deleteBlockRequest, context);
Mockito.verify(handler, times(1)).handleDeleteBlock(
any(ContainerCommandRequestProto.class), any());
// Test List Block Request handling
ContainerCommandRequestProto listBlockRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ListBlock);
- dispatcher.dispatch(listBlockRequest);
+ dispatcher.dispatch(listBlockRequest, context);
Mockito.verify(handler, times(2)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Read Chunk Request handling
ContainerCommandRequestProto readChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ReadChunk);
- dispatcher.dispatch(readChunkRequest);
+ dispatcher.dispatch(readChunkRequest, context);
Mockito.verify(handler, times(1)).handleReadChunk(
- any(ContainerCommandRequestProto.class), any());
+ any(ContainerCommandRequestProto.class), any(), any());
// Test Delete Chunk Request handling
ContainerCommandRequestProto deleteChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.DeleteChunk);
- dispatcher.dispatch(deleteChunkRequest);
+ dispatcher.dispatch(deleteChunkRequest, context);
Mockito.verify(handler, times(1)).handleDeleteChunk(
any(ContainerCommandRequestProto.class), any());
// Test Write Chunk Request handling
ContainerCommandRequestProto writeChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.WriteChunk);
- dispatcher.dispatch(writeChunkRequest);
+ dispatcher.dispatch(writeChunkRequest, context);
Mockito.verify(handler, times(1)).handleWriteChunk(
- any(ContainerCommandRequestProto.class), any());
+ any(ContainerCommandRequestProto.class), any(), any());
// Test List Chunk Request handling
ContainerCommandRequestProto listChunkRequest =
getDummyCommandRequestProto(ContainerProtos.Type.ListChunk);
- dispatcher.dispatch(listChunkRequest);
+ dispatcher.dispatch(listChunkRequest, context);
Mockito.verify(handler, times(3)).handleUnsupportedOp(
any(ContainerCommandRequestProto.class));
// Test Put Small File Request handling
ContainerCommandRequestProto putSmallFileRequest =
getDummyCommandRequestProto(ContainerProtos.Type.PutSmallFile);
- dispatcher.dispatch(putSmallFileRequest);
+ dispatcher.dispatch(putSmallFileRequest, context);
Mockito.verify(handler, times(1)).handlePutSmallFile(
- any(ContainerCommandRequestProto.class), any());
+ any(ContainerCommandRequestProto.class), any(), any());
// Test Get Small File Request handling
ContainerCommandRequestProto getSmallFileRequest =
getDummyCommandRequestProto(ContainerProtos.Type.GetSmallFile);
- dispatcher.dispatch(getSmallFileRequest);
+ dispatcher.dispatch(getSmallFileRequest, context);
Mockito.verify(handler, times(1)).handleGetSmallFile(
any(ContainerCommandRequestProto.class), any());
}
@@ -294,7 +296,7 @@ public class TestKeyValueHandler {
.setCloseContainer(ContainerProtos.CloseContainerRequestProto
.getDefaultInstance())
.build();
- dispatcher.dispatch(closeContainerRequest);
+ dispatcher.dispatch(closeContainerRequest, null);
Mockito.when(handler.handleCloseContainer(any(), any()))
.thenCallRealMethod();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 5d19a10..1637ff7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -78,7 +78,7 @@ import java.util.UUID;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.BCSID_MISMATCH;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNKNOWN_BCSID;
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Stage.COMBINED;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext.WriteChunkStage;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getData;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.setDataChecksum;
@@ -334,7 +334,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
return info;
}
@@ -375,7 +375,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
String fileName = String.format("%s.data.%d", blockID.getLocalID(), x);
fileHashMap.put(fileName, info);
}
@@ -433,7 +433,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
byte[] readData = chunkManager.readChunk(container, blockID, info, false);
assertTrue(Arrays.equals(data, readData));
@@ -466,13 +466,13 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
// With the overwrite flag it should work now.
info.addMetadata(OzoneConsts.CHUNK_OVERWRITE, "true");
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
long bytesUsed = container.getContainerData().getBytesUsed();
Assert.assertEquals(datalen, bytesUsed);
@@ -507,7 +507,7 @@ public class TestContainerPersistence {
oldSha.update(data);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
}
// Request to read the whole data in a single go.
@@ -540,7 +540,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
chunkManager.deleteChunk(container, blockID, info);
exception.expect(StorageContainerException.class);
exception.expectMessage("Unable to find the chunk file.");
@@ -655,7 +655,7 @@ public class TestContainerPersistence {
byte[] data = getData(datalen);
setDataChecksum(info, data);
chunkManager.writeChunk(container, blockID, info, ByteBuffer.wrap(data),
- COMBINED);
+ WriteChunkStage.COMBINED);
totalSize += datalen;
chunkList.add(info);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
index ab2ddf0..43606f0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
@@ -169,7 +169,8 @@ public class TestCSMMetrics {
*/
@Override
public ContainerCommandResponseProto dispatch(
- ContainerCommandRequestProto msg) {
+ ContainerCommandRequestProto msg,
+ DispatcherContext context) {
return ContainerTestHelper.getCreateContainerResponse(msg);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index 3e98594..1675484 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
@@ -236,8 +237,9 @@ public class TestContainerServer {
* @return Command Response
*/
@Override
- public ContainerCommandResponseProto
- dispatch(ContainerCommandRequestProto msg) {
+ public ContainerCommandResponseProto dispatch(
+ ContainerCommandRequestProto msg,
+ DispatcherContext context) {
return ContainerTestHelper.getCreateContainerResponse(msg);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3c7714/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 01b51fa..91d0968 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -130,7 +130,7 @@ public class BenchMarkDatanodeDispatcher {
for (int x = 0; x < INIT_CONTAINERS; x++) {
long containerID = HddsUtils.getUtcTime() + x;
ContainerCommandRequestProto req = getCreateContainerCommand(containerID);
- dispatcher.dispatch(req);
+ dispatcher.dispatch(req, null);
containers.add(containerID);
containerCount.getAndIncrement();
}
@@ -153,8 +153,8 @@ public class BenchMarkDatanodeDispatcher {
long containerID = containers.get(y);
BlockID blockID = new BlockID(containerID, key);
dispatcher
- .dispatch(getPutBlockCommand(blockID, chunkName));
- dispatcher.dispatch(getWriteChunkCommand(blockID, chunkName));
+ .dispatch(getPutBlockCommand(blockID, chunkName), null);
+ dispatcher.dispatch(getWriteChunkCommand(blockID, chunkName), null);
}
}
}
@@ -268,7 +268,7 @@ public class BenchMarkDatanodeDispatcher {
public void createContainer(BenchMarkDatanodeDispatcher bmdd) {
long containerID = RandomUtils.nextLong();
ContainerCommandRequestProto req = getCreateContainerCommand(containerID);
- bmdd.dispatcher.dispatch(req);
+ bmdd.dispatcher.dispatch(req, null);
bmdd.containers.add(containerID);
bmdd.containerCount.getAndIncrement();
}
@@ -277,27 +277,27 @@ public class BenchMarkDatanodeDispatcher {
@Benchmark
public void writeChunk(BenchMarkDatanodeDispatcher bmdd) {
bmdd.dispatcher.dispatch(getWriteChunkCommand(
- getRandomBlockID(), getNewChunkToWrite()));
+ getRandomBlockID(), getNewChunkToWrite()), null);
}
@Benchmark
public void readChunk(BenchMarkDatanodeDispatcher bmdd) {
BlockID blockID = getRandomBlockID();
String chunkKey = getRandomChunkToRead();
- bmdd.dispatcher.dispatch(getReadChunkCommand(blockID, chunkKey));
+ bmdd.dispatcher.dispatch(getReadChunkCommand(blockID, chunkKey), null);
}
@Benchmark
public void putBlock(BenchMarkDatanodeDispatcher bmdd) {
BlockID blockID = getRandomBlockID();
String chunkKey = getNewChunkToWrite();
- bmdd.dispatcher.dispatch(getPutBlockCommand(blockID, chunkKey));
+ bmdd.dispatcher.dispatch(getPutBlockCommand(blockID, chunkKey), null);
}
@Benchmark
public void getBlock(BenchMarkDatanodeDispatcher bmdd) {
BlockID blockID = getRandomBlockID();
- bmdd.dispatcher.dispatch(getGetBlockCommand(blockID));
+ bmdd.dispatcher.dispatch(getGetBlockCommand(blockID), null);
}
// Chunks writes from benchmark only reaches certain containers
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDFS-13713. Add specification of
Multipart Upload API to FS specification, with contract tests.
Posted by sh...@apache.org.
HDFS-13713. Add specification of Multipart Upload API to FS specification, with contract tests.
Contributed by Ewan Higgs and Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1d24f84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1d24f84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1d24f84
Branch: refs/heads/HDFS-12943
Commit: c1d24f848345f6d34a2ac2d570d49e9787a0df6a
Parents: b71cc7f
Author: Ewan Higgs <eh...@apache.org>
Authored: Thu Nov 29 15:11:07 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 29 15:12:17 2018 +0000
----------------------------------------------------------------------
.../hadoop/fs/FileSystemMultipartUploader.java | 36 +-
.../org/apache/hadoop/fs/MultipartUploader.java | 88 ++-
.../hadoop/fs/MultipartUploaderFactory.java | 7 +
.../src/site/markdown/filesystem/index.md | 1 +
.../markdown/filesystem/multipartuploader.md | 235 ++++++++
.../AbstractContractMultipartUploaderTest.java | 565 +++++++++++++++----
.../TestLocalFSContractMultipartUploader.java | 10 +
.../hdfs/TestHDFSContractMultipartUploader.java | 15 +
.../hadoop/fs/s3a/S3AMultipartUploader.java | 31 +-
.../s3a/ITestS3AContractMultipartUploader.java | 64 ++-
.../fs/s3a/TestS3AMultipartUploaderSupport.java | 2 +-
11 files changed, 876 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
index 94c7861..b77c244 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
@@ -19,21 +19,23 @@ package org.apache.hadoop.fs;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.stream.Collectors;
import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
import org.apache.commons.compress.utils.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.Path.mergePaths;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
* A MultipartUploader that uses the basic FileSystem commands.
@@ -70,7 +72,8 @@ public class FileSystemMultipartUploader extends MultipartUploader {
public PartHandle putPart(Path filePath, InputStream inputStream,
int partNumber, UploadHandle uploadId, long lengthInBytes)
throws IOException {
-
+ checkPutArguments(filePath, inputStream, partNumber, uploadId,
+ lengthInBytes);
byte[] uploadIdByteArray = uploadId.toByteArray();
checkUploadId(uploadIdByteArray);
Path collectorPath = new Path(new String(uploadIdByteArray, 0,
@@ -82,16 +85,17 @@ public class FileSystemMultipartUploader extends MultipartUploader {
fs.createFile(partPath).build()) {
IOUtils.copy(inputStream, fsDataOutputStream, 4096);
} finally {
- org.apache.hadoop.io.IOUtils.cleanupWithLogger(LOG, inputStream);
+ cleanupWithLogger(LOG, inputStream);
}
return BBPartHandle.from(ByteBuffer.wrap(
partPath.toString().getBytes(Charsets.UTF_8)));
}
private Path createCollectorPath(Path filePath) {
+ String uuid = UUID.randomUUID().toString();
return mergePaths(filePath.getParent(),
mergePaths(new Path(filePath.getName().split("\\.")[0]),
- mergePaths(new Path("_multipart"),
+ mergePaths(new Path("_multipart_" + uuid),
new Path(Path.SEPARATOR))));
}
@@ -110,21 +114,16 @@ public class FileSystemMultipartUploader extends MultipartUploader {
@Override
@SuppressWarnings("deprecation") // rename w/ OVERWRITE
- public PathHandle complete(Path filePath,
- List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
- throws IOException {
+ public PathHandle complete(Path filePath, Map<Integer, PartHandle> handleMap,
+ UploadHandle multipartUploadId) throws IOException {
checkUploadId(multipartUploadId.toByteArray());
- if (handles.isEmpty()) {
- throw new IOException("Empty upload");
- }
- // If destination already exists, we believe we already completed it.
- if (fs.exists(filePath)) {
- return getPathHandle(filePath);
- }
+ checkPartHandles(handleMap);
+ List<Map.Entry<Integer, PartHandle>> handles =
+ new ArrayList<>(handleMap.entrySet());
+ handles.sort(Comparator.comparingInt(Map.Entry::getKey));
- handles.sort(Comparator.comparing(Pair::getKey));
List<Path> partHandles = handles
.stream()
.map(pair -> {
@@ -134,7 +133,10 @@ public class FileSystemMultipartUploader extends MultipartUploader {
})
.collect(Collectors.toList());
- Path collectorPath = createCollectorPath(filePath);
+ byte[] uploadIdByteArray = multipartUploadId.toByteArray();
+ Path collectorPath = new Path(new String(uploadIdByteArray, 0,
+ uploadIdByteArray.length, Charsets.UTF_8));
+
boolean emptyFile = totalPartsLen(partHandles) == 0;
if (emptyFile) {
fs.create(filePath).close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
index b56dbf8..7ed987e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
@@ -17,40 +17,45 @@
*/
package org.apache.hadoop.fs;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
-import java.util.List;
+import java.util.Map;
-import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang3.tuple.Pair;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import static com.google.common.base.Preconditions.checkArgument;
+
/**
* MultipartUploader is an interface for copying files multipart and across
* multiple nodes. Users should:
* <ol>
- * <li>Initialize an upload</li>
- * <li>Upload parts in any order</li>
+ * <li>Initialize an upload.</li>
+ * <li>Upload parts in any order.</li>
* <li>Complete the upload in order to have it materialize in the destination
- * FS</li>
+ * FS.</li>
* </ol>
- *
- * Implementers should make sure that the complete function should make sure
- * that 'complete' will reorder parts if the destination FS doesn't already
- * do it for them.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public abstract class MultipartUploader {
+public abstract class MultipartUploader implements Closeable {
public static final Logger LOG =
LoggerFactory.getLogger(MultipartUploader.class);
/**
+ * Perform any cleanup.
+ * The upload is not required to support any operations after this.
+ * @throws IOException problems on close.
+ */
+ @Override
+ public void close() throws IOException {
+ }
+
+ /**
* Initialize a multipart upload.
* @param filePath Target path for upload.
* @return unique identifier associating part uploads.
@@ -59,8 +64,8 @@ public abstract class MultipartUploader {
public abstract UploadHandle initialize(Path filePath) throws IOException;
/**
- * Put part as part of a multipart upload. It should be possible to have
- * parts uploaded in any order (or in parallel).
+ * Put part as part of a multipart upload.
+ * It is possible to have parts uploaded in any order (or in parallel).
* @param filePath Target path for upload (same as {@link #initialize(Path)}).
* @param inputStream Data for this part. Implementations MUST close this
* stream after reading in the data.
@@ -77,15 +82,15 @@ public abstract class MultipartUploader {
/**
* Complete a multipart upload.
* @param filePath Target path for upload (same as {@link #initialize(Path)}.
- * @param handles non-empty list of identifiers with associated part numbers
+ * @param handles non-empty map of part number to part handle.
* from {@link #putPart(Path, InputStream, int, UploadHandle, long)}.
- * Depending on the backend, the list order may be significant.
* @param multipartUploadId Identifier from {@link #initialize(Path)}.
* @return unique PathHandle identifier for the uploaded file.
- * @throws IOException IO failure or the handle list is empty.
+ * @throws IOException IO failure
*/
public abstract PathHandle complete(Path filePath,
- List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
+ Map<Integer, PartHandle> handles,
+ UploadHandle multipartUploadId)
throws IOException;
/**
@@ -98,13 +103,52 @@ public abstract class MultipartUploader {
throws IOException;
/**
- * Utility method to validate uploadIDs
- * @param uploadId
- * @throws IllegalArgumentException
+ * Utility method to validate uploadIDs.
+ * @param uploadId Upload ID
+ * @throws IllegalArgumentException invalid ID
*/
protected void checkUploadId(byte[] uploadId)
throws IllegalArgumentException {
- Preconditions.checkArgument(uploadId.length > 0,
+ checkArgument(uploadId != null, "null uploadId");
+ checkArgument(uploadId.length > 0,
"Empty UploadId is not valid");
}
+
+ /**
+ * Utility method to validate partHandles.
+ * @param partHandles handles
+ * @throws IllegalArgumentException if the parts are invalid
+ */
+ protected void checkPartHandles(Map<Integer, PartHandle> partHandles) {
+ checkArgument(!partHandles.isEmpty(),
+ "Empty upload");
+ partHandles.keySet()
+ .stream()
+ .forEach(key ->
+ checkArgument(key > 0,
+ "Invalid part handle index %s", key));
+ }
+
+ /**
+ * Check all the arguments to the
+ * {@link #putPart(Path, InputStream, int, UploadHandle, long)} operation.
+ * @param filePath Target path for upload (same as {@link #initialize(Path)}).
+ * @param inputStream Data for this part. Implementations MUST close this
+ * stream after reading in the data.
+ * @param partNumber Index of the part relative to others.
+ * @param uploadId Identifier from {@link #initialize(Path)}.
+ * @param lengthInBytes Target length to read from the stream.
+ * @throws IllegalArgumentException invalid argument
+ */
+ protected void checkPutArguments(Path filePath,
+ InputStream inputStream,
+ int partNumber,
+ UploadHandle uploadId,
+ long lengthInBytes) throws IllegalArgumentException {
+ checkArgument(filePath != null, "null filePath");
+ checkArgument(inputStream != null, "null inputStream");
+ checkArgument(partNumber > 0, "Invalid part number: %d", partNumber);
+ checkArgument(uploadId != null, "null uploadId");
+ checkArgument(lengthInBytes >= 0, "Invalid part length: %d", lengthInBytes);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
index 8b35232..e35b6bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderFactory.java
@@ -52,6 +52,13 @@ public abstract class MultipartUploaderFactory {
}
}
+ /**
+ * Get the multipart loader for a specific filesystem.
+ * @param fs filesystem
+ * @param conf configuration
+ * @return an uploader, or null if one was found.
+ * @throws IOException failure during the creation process.
+ */
public static MultipartUploader get(FileSystem fs, Configuration conf)
throws IOException {
MultipartUploader mpu = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
index 532b6c7..6b4399e 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
@@ -36,3 +36,4 @@ HDFS as these are commonly expected by Hadoop client applications.
1. [FSDataOutputStreamBuilder class](fsdataoutputstreambuilder.html)
2. [Testing with the Filesystem specification](testing.html)
2. [Extending the specification and its tests](extending.html)
+1. [Uploading a file using Multiple Parts](multipartuploader.html)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
new file mode 100644
index 0000000..629c0c4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/multipartuploader.md
@@ -0,0 +1,235 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+<!-- ============================================================= -->
+<!-- CLASS: MultipartUploader -->
+<!-- ============================================================= -->
+
+# class `org.apache.hadoop.fs.MultipartUploader`
+
+<!-- MACRO{toc|fromDepth=1|toDepth=2} -->
+
+The abstract `MultipartUploader` class is the original class to upload a file
+using multiple parts to Hadoop-supported filesystems. The benefits of a
+multipart upload is that the file can be uploaded from multiple clients or
+processes in parallel and the results will not be visible to other clients until
+the `complete` function is called.
+
+When implemented by an object store, uploaded data may incur storage charges,
+even before it is visible in the filesystems. Users of this API must be diligent
+and always perform best-effort attempts to complete or abort the upload.
+
+## Invariants
+
+All the requirements of a valid MultipartUploader are considered implicit
+econditions and postconditions:
+all operations on a valid MultipartUploader MUST result in a new
+MultipartUploader that is also valid.
+
+The operations of a single multipart upload may take place across different
+instance of a multipart uploader, across different processes and hosts.
+It is therefore a requirement that:
+
+1. All state needed to upload a part, complete an upload or abort an upload
+must be contained within or retrievable from an upload handle.
+
+1. If an upload handle is marshalled to another process, then, if the
+receiving process has the correct permissions, it may participate in the
+upload, by uploading one or more parts, by completing an upload, and/or by
+aborting the upload.
+
+## Concurrency
+
+Multiple processes may upload parts of a multipart upload simultaneously.
+
+If a call is made to `initialize(path)` to a destination where an active
+upload is in progress, implementations MUST perform one of the two operations.
+
+* Reject the call as a duplicate.
+* Permit both to proceed, with the final output of the file being
+that of _exactly one of the two uploads_.
+
+Which upload succeeds is undefined. Users must not expect consistent
+behavior across filesystems, across filesystem instances *or even
+across different requests.
+
+If a multipart upload is completed or aborted while a part upload is in progress,
+the in-progress upload, if it has not completed, must not be included in
+the final file, in whole or in part. Implementations SHOULD raise an error
+in the `putPart()` operation.
+
+## Model
+
+A File System which supports Multipart Uploads extends the existing model
+`(Directories, Files, Symlinks)` to one of `(Directories, Files, Symlinks, Uploads)`
+`Uploads` of type `Map[UploadHandle -> Map[PartHandle -> UploadPart]`.
+
+
+The `Uploads` element of the state tuple is a map of all active uploads.
+
+```python
+Uploads: Map[UploadHandle -> Map[PartHandle -> UploadPart]`
+```
+
+An UploadHandle is a non-empty list of bytes.
+```python
+UploadHandle: List[byte]
+len(UploadHandle) > 0
+```
+
+Clients *MUST* treat this as opaque. What is core to this features design is that the handle is valid from
+across clients: the handle may be serialized on host `hostA`, deserialized on `hostB` and still used
+to extend or complete the upload.
+
+```python
+UploadPart = (Path: path, parts: Map[PartHandle -> byte[]])
+```
+
+Similarly, the `PartHandle` type is also a non-empty list of opaque bytes, again, marshallable between hosts.
+
+```python
+PartHandle: List[byte]
+```
+
+It is implicit that each `UploadHandle` in `FS.Uploads` is unique.
+Similarly, each `PartHandle` in the map of `[PartHandle -> UploadPart]` must also be unique.
+
+1. There is no requirement that Part Handles are unique across uploads.
+1. There is no requirement that Upload Handles are unique over time.
+However, if Part Handles are rapidly recycled, there is a risk that the nominally
+idempotent operation `abort(FS, uploadHandle)` could unintentionally cancel a
+successor operation which used the same Upload Handle.
+
+## State Changing Operations
+
+### `UploadHandle initialize(Path path)`
+
+Initialized a Multipart Upload, returning an upload handle for use in
+subsequent operations.
+
+#### Preconditions
+
+```python
+if path == "/" : raise IOException
+
+if exists(FS, path) and not isFile(FS, path) raise PathIsDirectoryException, IOException
+```
+
+If a filesystem does not support concurrent uploads to a destination,
+then the following precondition is added
+
+```python
+if path in values(FS.Uploads) raise PathExistsException, IOException
+
+```
+
+
+#### Postconditions
+
+The outcome of this operation is that the filesystem state is updated with a new
+active upload, with a new handle, this handle being returned to the caller.
+
+```python
+handle' = UploadHandle where not handle' in keys(FS.Uploads)
+FS' = FS where FS'.Uploads(handle') == {}
+result = handle'
+```
+
+### `PartHandle putPart(Path path, InputStream inputStream, int partNumber, UploadHandle uploadHandle, long lengthInBytes)`
+
+Upload a part for the multipart upload.
+
+#### Preconditions
+
+
+```python
+uploadHandle in keys(FS.Uploads)
+partNumber >= 1
+lengthInBytes >= 0
+len(inputStream) >= lengthInBytes
+```
+
+#### Postconditions
+
+```python
+data' = inputStream(0..lengthInBytes)
+partHandle' = byte[] where not partHandle' in keys(FS.uploads(uploadHandle).parts)
+FS' = FS where FS'.uploads(uploadHandle).parts(partHandle') == data'
+result = partHandle'
+```
+
+The data is stored in the filesystem, pending completion.
+
+
+### `PathHandle complete(Path path, Map<Integer, PartHandle> parts, UploadHandle multipartUploadId)`
+
+Complete the multipart upload.
+
+A Filesystem may enforce a minimum size of each part, excluding the last part uploaded.
+
+If a part is out of this range, an `IOException` MUST be raised.
+
+#### Preconditions
+
+```python
+uploadHandle in keys(FS.Uploads) else raise FileNotFoundException
+FS.Uploads(uploadHandle).path == path
+if exists(FS, path) and not isFile(FS, path) raise PathIsDirectoryException, IOException
+parts.size() > 0
+```
+
+If there are handles in the MPU which aren't included in the map, then the omitted
+parts will not be a part of the resulting file. It is up to the implementation
+of the MultipartUploader to make sure the leftover parts are cleaned up.
+
+In the case of backing stores that support directories (local filesystem, HDFS,
+etc), if, at the point of completion, there is now a directory at the
+destination then a `PathIsDirectoryException` or other `IOException` must be thrown.
+
+#### Postconditions
+
+```python
+UploadData' == ordered concatention of all data in the map of parts, ordered by key
+exists(FS', path') and result = PathHandle(path')
+FS' = FS where FS.Files(path) == UploadData' and not uploadHandle in keys(FS'.uploads)
+```
+
+The PathHandle is returned by the complete operation so subsequent operations
+will be able to identify that the data has not changed in the meantime.
+
+The order of parts in the uploaded by file is that of the natural order of
+parts: part 1 is ahead of part 2, etc.
+
+
+### `void abort(Path path, UploadHandle multipartUploadId)`
+
+Abort a multipart upload. The handle becomes invalid and not subject to reuse.
+
+#### Preconditions
+
+
+```python
+uploadHandle in keys(FS.Uploads) else raise FileNotFoundException
+```
+
+#### Postconditions
+
+The upload handle is no longer known.
+
+```python
+FS' = FS where not uploadHandle in keys(FS'.uploads)
+```
+A subsequent call to `abort()` with the same handle will fail, unless
+the handle has been recycled.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
index 7cee5a6..7a8f083 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
@@ -23,15 +23,19 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
import com.google.common.base.Charsets;
+import org.junit.Assume;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BBUploadHandle;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -43,11 +47,64 @@ import org.apache.hadoop.fs.PathHandle;
import org.apache.hadoop.fs.UploadHandle;
import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
+import static org.apache.hadoop.test.LambdaTestUtils.eventually;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
public abstract class AbstractContractMultipartUploaderTest extends
AbstractFSContractTestBase {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(AbstractContractMultipartUploaderTest.class);
+
+ /**
+ * Size of very small uploads.
+ * Enough to be non empty, not big enough to cause delays on uploads.
+ */
+ protected static final int SMALL_FILE = 100;
+
+ private MultipartUploader mpu;
+ private MultipartUploader mpu2;
+ private final Random random = new Random();
+ private UploadHandle activeUpload;
+ private Path activeUploadPath;
+
+ protected String getMethodName() {
+ return methodName.getMethodName();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ Configuration conf = getContract().getConf();
+ mpu = MultipartUploaderFactory.get(getFileSystem(), conf);
+ mpu2 = MultipartUploaderFactory.get(getFileSystem(), conf);
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ if (mpu!= null && activeUpload != null) {
+ try {
+ mpu.abort(activeUploadPath, activeUpload);
+ } catch (FileNotFoundException ignored) {
+ /* this is fine */
+ } catch (Exception e) {
+ LOG.info("in teardown", e);
+ }
+ }
+ cleanupWithLogger(LOG, mpu, mpu2);
+ super.teardown();
+ }
+
+ /**
+ * Get a test path based on the method name.
+ * @return a path to use in the test
+ * @throws IOException failure to build the path name up.
+ */
+ protected Path methodPath() throws IOException {
+ return path(getMethodName());
+ }
+
/**
* The payload is the part number repeated for the length of the part.
* This makes checking the correctness of the upload straightforward.
@@ -55,9 +112,19 @@ public abstract class AbstractContractMultipartUploaderTest extends
* @return the bytes to upload.
*/
private byte[] generatePayload(int partNumber) {
- int sizeInBytes = partSizeInBytes();
- ByteBuffer buffer = ByteBuffer.allocate(sizeInBytes);
- for (int i=0 ; i < sizeInBytes/(Integer.SIZE/Byte.SIZE); ++i) {
+ return generatePayload(partNumber, partSizeInBytes());
+ }
+
+ /**
+ * Generate a payload of a given size; part number is used
+ * for all the values.
+ * @param partNumber part number
+ * @param size size in bytes
+ * @return the bytes to upload.
+ */
+ private byte[] generatePayload(final int partNumber, final int size) {
+ ByteBuffer buffer = ByteBuffer.allocate(size);
+ for (int i=0; i < size /(Integer.SIZE/Byte.SIZE); ++i) {
buffer.putInt(partNumber);
}
return buffer.array();
@@ -70,11 +137,14 @@ public abstract class AbstractContractMultipartUploaderTest extends
* @throws IOException failure to read or digest the file.
*/
protected byte[] digest(Path path) throws IOException {
- FileSystem fs = getFileSystem();
- try (InputStream in = fs.open(path)) {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ try (InputStream in = getFileSystem().open(path)) {
byte[] fdData = IOUtils.toByteArray(in);
MessageDigest newDigest = DigestUtils.getMd5Digest();
- return newDigest.digest(fdData);
+ byte[] digest = newDigest.digest(fdData);
+ return digest;
+ } finally {
+ timer.end("Download and digest of path %s", path);
}
}
@@ -93,74 +163,230 @@ public abstract class AbstractContractMultipartUploaderTest extends
}
/**
+ * How long in milliseconds for propagation of
+ * store changes, including update/delete/list
+ * to be everywhere.
+ * If 0: the FS is consistent.
+ * @return a time in milliseconds.
+ */
+ protected int timeToBecomeConsistentMillis() {
+ return 0;
+ }
+
+ /**
+ * Does a call to finalize an upload (either complete or abort) consume the
+ * uploadID immediately or is it reaped at a later point in time?
+ * @return true if the uploadID will be consumed immediately (and no longer
+ * resuable).
+ */
+ protected abstract boolean finalizeConsumesUploadIdImmediately();
+
+ /**
+ * Does the store support concurrent uploads to the same destination path?
+ * @return true if concurrent uploads are supported.
+ */
+ protected abstract boolean supportsConcurrentUploadsToSamePath();
+
+ /**
+ * Pick a multipart uploader from the index value.
+ * @param index index of upload
+ * @return an uploader
+ */
+ protected MultipartUploader mpu(int index) {
+ return (index % 2 == 0) ? mpu : mpu2;
+ }
+
+ /**
+ * Pick a multipart uploader at random.
+ * @return an uploader
+ */
+ protected MultipartUploader randomMpu() {
+ return mpu(random.nextInt(10));
+ }
+
+ /**
* Assert that a multipart upload is successful.
* @throws Exception failure
*/
@Test
public void testSingleUpload() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testSingleUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
MessageDigest origDigest = DigestUtils.getMd5Digest();
- byte[] payload = generatePayload(1);
+ int size = SMALL_FILE;
+ byte[] payload = generatePayload(1, size);
origDigest.update(payload);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, 1, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(1, partHandle));
- PathHandle fd = completeUpload(file, mpu, uploadHandle, partHandles,
+ PartHandle partHandle = putPart(file, uploadHandle, 1, payload);
+ partHandles.put(1, partHandle);
+ PathHandle fd = completeUpload(file, uploadHandle, partHandles,
origDigest,
- payload.length);
+ size);
+
+ if (finalizeConsumesUploadIdImmediately()) {
+ intercept(FileNotFoundException.class,
+ () -> mpu.complete(file, partHandles, uploadHandle));
+ } else {
+ PathHandle fd2 = mpu.complete(file, partHandles, uploadHandle);
+ assertArrayEquals("Path handles differ", fd.toByteArray(),
+ fd2.toByteArray());
+ }
+ }
+
+ /**
+ * Initialize an upload.
+ * This saves the path and upload handle as the active
+ * upload, for aborting in teardown
+ * @param dest destination
+ * @return the handle
+ * @throws IOException failure to initialize
+ */
+ protected UploadHandle initializeUpload(final Path dest) throws IOException {
+ activeUploadPath = dest;
+ activeUpload = randomMpu().initialize(dest);
+ return activeUpload;
+ }
- // Complete is idempotent
- PathHandle fd2 = mpu.complete(file, partHandles, uploadHandle);
- assertArrayEquals("Path handles differ", fd.toByteArray(),
- fd2.toByteArray());
+ /**
+ * Generate then upload a part.
+ * @param file destination
+ * @param uploadHandle handle
+ * @param index index of part
+ * @param origDigest digest to build up. May be null
+ * @return the part handle
+ * @throws IOException IO failure.
+ */
+ protected PartHandle buildAndPutPart(
+ final Path file,
+ final UploadHandle uploadHandle,
+ final int index,
+ final MessageDigest origDigest) throws IOException {
+ byte[] payload = generatePayload(index);
+ if (origDigest != null) {
+ origDigest.update(payload);
+ }
+ return putPart(file, uploadHandle, index, payload);
}
+ /**
+ * Put a part.
+ * The entire byte array is uploaded.
+ * @param file destination
+ * @param uploadHandle handle
+ * @param index index of part
+ * @param payload byte array of payload
+ * @return the part handle
+ * @throws IOException IO failure.
+ */
+ protected PartHandle putPart(final Path file,
+ final UploadHandle uploadHandle,
+ final int index,
+ final byte[] payload) throws IOException {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ PartHandle partHandle = mpu(index)
+ .putPart(file,
+ new ByteArrayInputStream(payload),
+ index,
+ uploadHandle,
+ payload.length);
+ timer.end("Uploaded part %s", index);
+ LOG.info("Upload bandwidth {} MB/s",
+ timer.bandwidthDescription(payload.length));
+ return partHandle;
+ }
+
+ /**
+ * Complete an upload with the active MPU instance.
+ * @param file destination
+ * @param uploadHandle handle
+ * @param partHandles map of handles
+ * @param origDigest digest of source data (may be null)
+ * @param expectedLength expected length of result.
+ * @return the path handle from the upload.
+ * @throws IOException IO failure
+ */
private PathHandle completeUpload(final Path file,
- final MultipartUploader mpu,
final UploadHandle uploadHandle,
- final List<Pair<Integer, PartHandle>> partHandles,
+ final Map<Integer, PartHandle> partHandles,
final MessageDigest origDigest,
final int expectedLength) throws IOException {
- PathHandle fd = mpu.complete(file, partHandles, uploadHandle);
+ PathHandle fd = complete(file, uploadHandle, partHandles);
FileStatus status = verifyPathExists(getFileSystem(),
"Completed file", file);
assertEquals("length of " + status,
expectedLength, status.getLen());
+ if (origDigest != null) {
+ verifyContents(file, origDigest, expectedLength);
+ }
+ return fd;
+ }
+
+ /**
+ * Verify the contents of a file.
+ * @param file path
+ * @param origDigest digest
+ * @param expectedLength expected length (for logging B/W)
+ * @throws IOException IO failure
+ */
+ protected void verifyContents(final Path file,
+ final MessageDigest origDigest,
+ final int expectedLength) throws IOException {
+ ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
assertArrayEquals("digest of source and " + file
+ " differ",
origDigest.digest(), digest(file));
+ timer2.end("Completed digest", file);
+ LOG.info("Download bandwidth {} MB/s",
+ timer2.bandwidthDescription(expectedLength));
+ }
+
+ /**
+ * Perform the inner complete without verification.
+ * @param file destination path
+ * @param uploadHandle upload handle
+ * @param partHandles map of parts
+ * @return the path handle from the upload.
+ * @throws IOException IO failure
+ */
+ private PathHandle complete(final Path file,
+ final UploadHandle uploadHandle,
+ final Map<Integer, PartHandle> partHandles) throws IOException {
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ PathHandle fd = randomMpu().complete(file, partHandles, uploadHandle);
+ timer.end("Completed upload to %s", file);
return fd;
}
/**
+ * Abort an upload.
+ * @param file path
+ * @param uploadHandle handle
+ * @throws IOException failure
+ */
+ private void abortUpload(final Path file, UploadHandle uploadHandle)
+ throws IOException {
+ randomMpu().abort(file, uploadHandle);
+ }
+
+ /**
* Assert that a multipart upload is successful.
* @throws Exception failure
*/
@Test
public void testMultipartUpload() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
MessageDigest origDigest = DigestUtils.getMd5Digest();
final int payloadCount = getTestPayloadCount();
for (int i = 1; i <= payloadCount; ++i) {
- byte[] payload = generatePayload(i);
- origDigest.update(payload);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ PartHandle partHandle = buildAndPutPart(file, uploadHandle, i,
+ origDigest);
+ partHandles.put(i, partHandle);
}
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+ completeUpload(file, uploadHandle, partHandles, origDigest,
payloadCount * partSizeInBytes());
}
@@ -173,17 +399,33 @@ public abstract class AbstractContractMultipartUploaderTest extends
public void testMultipartUploadEmptyPart() throws Exception {
FileSystem fs = getFileSystem();
Path file = path("testMultipartUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
- MessageDigest origDigest = DigestUtils.getMd5Digest();
- byte[] payload = new byte[0];
- origDigest.update(payload);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, 0, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(0, partHandle));
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest, 0);
+ try (MultipartUploader uploader =
+ MultipartUploaderFactory.get(fs, null)) {
+ UploadHandle uploadHandle = uploader.initialize(file);
+
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ MessageDigest origDigest = DigestUtils.getMd5Digest();
+ byte[] payload = new byte[0];
+ origDigest.update(payload);
+ InputStream is = new ByteArrayInputStream(payload);
+ PartHandle partHandle = uploader.putPart(file, is, 1, uploadHandle,
+ payload.length);
+ partHandles.put(1, partHandle);
+ completeUpload(file, uploadHandle, partHandles, origDigest, 0);
+ }
+ }
+
+ /**
+ * Assert that a multipart upload is successful.
+ * @throws Exception failure
+ */
+ @Test
+ public void testUploadEmptyBlock() throws Exception {
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ partHandles.put(1, putPart(file, uploadHandle, 1, new byte[0]));
+ completeUpload(file, uploadHandle, partHandles, null, 0);
}
/**
@@ -192,11 +434,9 @@ public abstract class AbstractContractMultipartUploaderTest extends
*/
@Test
public void testMultipartUploadReverseOrder() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUploadReverseOrder");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
MessageDigest origDigest = DigestUtils.getMd5Digest();
final int payloadCount = getTestPayloadCount();
for (int i = 1; i <= payloadCount; ++i) {
@@ -204,13 +444,9 @@ public abstract class AbstractContractMultipartUploaderTest extends
origDigest.update(payload);
}
for (int i = payloadCount; i > 0; --i) {
- byte[] payload = generatePayload(i);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
}
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+ completeUpload(file, uploadHandle, partHandles, origDigest,
payloadCount * partSizeInBytes());
}
@@ -222,25 +458,19 @@ public abstract class AbstractContractMultipartUploaderTest extends
public void testMultipartUploadReverseOrderNonContiguousPartNumbers()
throws Exception {
describe("Upload in reverse order and the part numbers are not contiguous");
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUploadReverseOrderNonContiguousPartNumbers");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
MessageDigest origDigest = DigestUtils.getMd5Digest();
int payloadCount = 2 * getTestPayloadCount();
for (int i = 2; i <= payloadCount; i += 2) {
byte[] payload = generatePayload(i);
origDigest.update(payload);
}
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
for (int i = payloadCount; i > 0; i -= 2) {
- byte[] payload = generatePayload(i);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
}
- completeUpload(file, mpu, uploadHandle, partHandles, origDigest,
+ completeUpload(file, uploadHandle, partHandles, origDigest,
getTestPayloadCount() * partSizeInBytes());
}
@@ -251,19 +481,14 @@ public abstract class AbstractContractMultipartUploaderTest extends
@Test
public void testMultipartUploadAbort() throws Exception {
describe("Upload and then abort it before completing");
- FileSystem fs = getFileSystem();
- Path file = path("testMultipartUploadAbort");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle uploadHandle = mpu.initialize(file);
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
- for (int i = 20; i >= 10; --i) {
- byte[] payload = generatePayload(i);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(file, is, i, uploadHandle,
- payload.length);
- partHandles.add(Pair.of(i, partHandle));
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ int end = 10;
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ for (int i = 12; i > 10; i--) {
+ partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null));
}
- mpu.abort(file, uploadHandle);
+ abortUpload(file, uploadHandle);
String contents = "ThisIsPart49\n";
int len = contents.getBytes(Charsets.UTF_8).length;
@@ -275,6 +500,15 @@ public abstract class AbstractContractMultipartUploaderTest extends
() -> mpu.complete(file, partHandles, uploadHandle));
assertPathDoesNotExist("Uploaded file should not exist", file);
+
+ // A second abort should be an FileNotFoundException if the UploadHandle is
+ // consumed by finalization operations (complete, abort).
+ if (finalizeConsumesUploadIdImmediately()) {
+ intercept(FileNotFoundException.class,
+ () -> abortUpload(file, uploadHandle));
+ } else {
+ abortUpload(file, uploadHandle);
+ }
}
/**
@@ -282,13 +516,23 @@ public abstract class AbstractContractMultipartUploaderTest extends
*/
@Test
public void testAbortUnknownUpload() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testAbortUnknownUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ Path file = methodPath();
ByteBuffer byteBuffer = ByteBuffer.wrap(
"invalid-handle".getBytes(Charsets.UTF_8));
UploadHandle uploadHandle = BBUploadHandle.from(byteBuffer);
- intercept(FileNotFoundException.class, () -> mpu.abort(file, uploadHandle));
+ intercept(FileNotFoundException.class,
+ () -> abortUpload(file, uploadHandle));
+ }
+
+ /**
+ * Trying to abort with a handle of size 0 must fail.
+ */
+ @Test
+ public void testAbortEmptyUpload() throws Exception {
+ describe("initialize upload and abort before uploading data");
+ Path file = methodPath();
+ abortUpload(file, initializeUpload(file));
+ assertPathDoesNotExist("Uploaded file should not exist", file);
}
/**
@@ -296,13 +540,10 @@ public abstract class AbstractContractMultipartUploaderTest extends
*/
@Test
public void testAbortEmptyUploadHandle() throws Exception {
- FileSystem fs = getFileSystem();
- Path file = path("testAbortEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[0]);
UploadHandle uploadHandle = BBUploadHandle.from(byteBuffer);
intercept(IllegalArgumentException.class,
- () -> mpu.abort(file, uploadHandle));
+ () -> abortUpload(methodPath(), uploadHandle));
}
/**
@@ -311,26 +552,20 @@ public abstract class AbstractContractMultipartUploaderTest extends
@Test
public void testCompleteEmptyUpload() throws Exception {
describe("Expect an empty MPU to fail, but still be abortable");
- FileSystem fs = getFileSystem();
- Path dest = path("testCompleteEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle handle = mpu.initialize(dest);
- intercept(IOException.class,
- () -> mpu.complete(dest, new ArrayList<>(), handle));
- mpu.abort(dest, handle);
+ Path dest = methodPath();
+ UploadHandle handle = initializeUpload(dest);
+ intercept(IllegalArgumentException.class,
+ () -> mpu.complete(dest, new HashMap<>(), handle));
+ abortUpload(dest, handle);
}
/**
* When we pass empty uploadID, putPart throws IllegalArgumentException.
- * @throws Exception
*/
@Test
public void testPutPartEmptyUploadID() throws Exception {
describe("Expect IllegalArgumentException when putPart uploadID is empty");
- FileSystem fs = getFileSystem();
- Path dest = path("testCompleteEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- mpu.initialize(dest);
+ Path dest = methodPath();
UploadHandle emptyHandle =
BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
byte[] payload = generatePayload(1);
@@ -341,25 +576,123 @@ public abstract class AbstractContractMultipartUploaderTest extends
/**
* When we pass empty uploadID, complete throws IllegalArgumentException.
- * @throws Exception
*/
@Test
public void testCompleteEmptyUploadID() throws Exception {
describe("Expect IllegalArgumentException when complete uploadID is empty");
- FileSystem fs = getFileSystem();
- Path dest = path("testCompleteEmptyUpload");
- MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
- UploadHandle realHandle = mpu.initialize(dest);
+ Path dest = methodPath();
+ UploadHandle realHandle = initializeUpload(dest);
UploadHandle emptyHandle =
BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
- List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
- byte[] payload = generatePayload(1);
- InputStream is = new ByteArrayInputStream(payload);
- PartHandle partHandle = mpu.putPart(dest, is, 1, realHandle,
- payload.length);
- partHandles.add(Pair.of(1, partHandle));
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ PartHandle partHandle = putPart(dest, realHandle, 1,
+ generatePayload(1, SMALL_FILE));
+ partHandles.put(1, partHandle);
intercept(IllegalArgumentException.class,
() -> mpu.complete(dest, partHandles, emptyHandle));
+
+ // and, while things are setup, attempt to complete with
+ // a part index of 0
+ partHandles.clear();
+ partHandles.put(0, partHandle);
+ intercept(IllegalArgumentException.class,
+ () -> mpu.complete(dest, partHandles, realHandle));
+ }
+
+ /**
+ * Assert that upon completion, a directory in the way of the file will
+ * result in a failure. This test only applies to backing stores with a
+ * concept of directories.
+ * @throws Exception failure
+ */
+ @Test
+ public void testDirectoryInTheWay() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path file = methodPath();
+ UploadHandle uploadHandle = initializeUpload(file);
+ Map<Integer, PartHandle> partHandles = new HashMap<>();
+ int size = SMALL_FILE;
+ PartHandle partHandle = putPart(file, uploadHandle, 1,
+ generatePayload(1, size));
+ partHandles.put(1, partHandle);
+
+ fs.mkdirs(file);
+ intercept(IOException.class,
+ () -> completeUpload(file, uploadHandle, partHandles, null,
+ size));
+ // abort should still work
+ abortUpload(file, uploadHandle);
+ }
+
+ @Test
+ public void testConcurrentUploads() throws Throwable {
+
+ // if the FS doesn't support concurrent uploads, this test is
+ // required to fail during the second initialization.
+ final boolean concurrent = supportsConcurrentUploadsToSamePath();
+
+ describe("testing concurrent uploads, MPU support for this is "
+ + concurrent);
+ final FileSystem fs = getFileSystem();
+ final Path file = methodPath();
+ final int size1 = SMALL_FILE;
+ final int partId1 = 1;
+ final byte[] payload1 = generatePayload(partId1, size1);
+ final MessageDigest digest1 = DigestUtils.getMd5Digest();
+ digest1.update(payload1);
+ final UploadHandle upload1 = initializeUpload(file);
+ final Map<Integer, PartHandle> partHandles1 = new HashMap<>();
+
+ // initiate part 2
+ // by using a different size, it's straightforward to see which
+ // version is visible, before reading/digesting the contents
+ final int size2 = size1 * 2;
+ final int partId2 = 2;
+ final byte[] payload2 = generatePayload(partId1, size2);
+ final MessageDigest digest2 = DigestUtils.getMd5Digest();
+ digest2.update(payload2);
+
+ final UploadHandle upload2;
+ try {
+ upload2 = initializeUpload(file);
+ Assume.assumeTrue(
+ "The Filesystem is unexpectedly supporting concurrent uploads",
+ concurrent);
+ } catch (IOException e) {
+ if (!concurrent) {
+ // this is expected, so end the test
+ LOG.debug("Expected exception raised on concurrent uploads {}", e);
+ return;
+ } else {
+ throw e;
+ }
+ }
+ final Map<Integer, PartHandle> partHandles2 = new HashMap<>();
+
+
+ assertNotEquals("Upload handles match", upload1, upload2);
+
+ // put part 1
+ partHandles1.put(partId1, putPart(file, upload1, partId1, payload1));
+
+ // put part2
+ partHandles2.put(partId2, putPart(file, upload2, partId2, payload2));
+
+ // complete part u1. expect its size and digest to
+ // be as expected.
+ completeUpload(file, upload1, partHandles1, digest1, size1);
+
+ // now upload part 2.
+ complete(file, upload2, partHandles2);
+ // and await the visible length to match
+ eventually(timeToBecomeConsistentMillis(), 500,
+ () -> {
+ FileStatus status = fs.getFileStatus(file);
+ assertEquals("File length in " + status,
+ size2, status.getLen());
+ });
+
+ verifyContents(file, digest2, size2);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
index a50d2e4..6e27964d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMultipartUploader.java
@@ -40,4 +40,14 @@ public class TestLocalFSContractMultipartUploader
protected int partSizeInBytes() {
return 1024;
}
+
+ @Override
+ protected boolean finalizeConsumesUploadIdImmediately() {
+ return true;
+ }
+
+ @Override
+ protected boolean supportsConcurrentUploadsToSamePath() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
index f3a5265..54f4ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMultipartUploader.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractMultipartUploaderTest;
@@ -32,6 +34,9 @@ import org.apache.hadoop.fs.contract.AbstractFSContract;
public class TestHDFSContractMultipartUploader extends
AbstractContractMultipartUploaderTest {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(AbstractContractMultipartUploaderTest.class);
+
@BeforeClass
public static void createCluster() throws IOException {
HDFSContract.createCluster();
@@ -55,4 +60,14 @@ public class TestHDFSContractMultipartUploader extends
protected int partSizeInBytes() {
return 1024;
}
+
+ @Override
+ protected boolean finalizeConsumesUploadIdImmediately() {
+ return true;
+ }
+
+ @Override
+ protected boolean supportsConcurrentUploadsToSamePath() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
index cab4e2a..cf58751 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
@@ -25,7 +25,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
@@ -68,10 +70,8 @@ public class S3AMultipartUploader extends MultipartUploader {
public static final String HEADER = "S3A-part01";
public S3AMultipartUploader(FileSystem fs, Configuration conf) {
- if (!(fs instanceof S3AFileSystem)) {
- throw new IllegalArgumentException(
- "S3A MultipartUploads must use S3AFileSystem");
- }
+ Preconditions.checkArgument(fs instanceof S3AFileSystem,
+ "Wrong filesystem: expected S3A but got %s", fs);
s3a = (S3AFileSystem) fs;
}
@@ -88,6 +88,8 @@ public class S3AMultipartUploader extends MultipartUploader {
public PartHandle putPart(Path filePath, InputStream inputStream,
int partNumber, UploadHandle uploadId, long lengthInBytes)
throws IOException {
+ checkPutArguments(filePath, inputStream, partNumber, uploadId,
+ lengthInBytes);
byte[] uploadIdBytes = uploadId.toByteArray();
checkUploadId(uploadIdBytes);
String key = s3a.pathToKey(filePath);
@@ -105,14 +107,16 @@ public class S3AMultipartUploader extends MultipartUploader {
@Override
public PathHandle complete(Path filePath,
- List<Pair<Integer, PartHandle>> handles, UploadHandle uploadId)
+ Map<Integer, PartHandle> handleMap,
+ UploadHandle uploadId)
throws IOException {
byte[] uploadIdBytes = uploadId.toByteArray();
checkUploadId(uploadIdBytes);
- if (handles.isEmpty()) {
- throw new IOException("Empty upload");
- }
+ checkPartHandles(handleMap);
+ List<Map.Entry<Integer, PartHandle>> handles =
+ new ArrayList<>(handleMap.entrySet());
+ handles.sort(Comparator.comparingInt(Map.Entry::getKey));
final WriteOperationHelper writeHelper = s3a.getWriteOperationHelper();
String key = s3a.pathToKey(filePath);
@@ -121,11 +125,11 @@ public class S3AMultipartUploader extends MultipartUploader {
ArrayList<PartETag> eTags = new ArrayList<>();
eTags.ensureCapacity(handles.size());
long totalLength = 0;
- for (Pair<Integer, PartHandle> handle : handles) {
- byte[] payload = handle.getRight().toByteArray();
+ for (Map.Entry<Integer, PartHandle> handle : handles) {
+ byte[] payload = handle.getValue().toByteArray();
Pair<Long, String> result = parsePartHandlePayload(payload);
totalLength += result.getLeft();
- eTags.add(new PartETag(handle.getLeft(), result.getRight()));
+ eTags.add(new PartETag(handle.getKey(), result.getRight()));
}
AtomicInteger errorCount = new AtomicInteger(0);
CompleteMultipartUploadResult result = writeHelper.completeMPUwithRetries(
@@ -172,7 +176,7 @@ public class S3AMultipartUploader extends MultipartUploader {
throws IOException {
Preconditions.checkArgument(StringUtils.isNotEmpty(eTag),
"Empty etag");
- Preconditions.checkArgument(len > 0,
+ Preconditions.checkArgument(len >= 0,
"Invalid length");
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@@ -190,6 +194,7 @@ public class S3AMultipartUploader extends MultipartUploader {
* @return the length and etag
* @throws IOException error reading the payload
*/
+ @VisibleForTesting
static Pair<Long, String> parsePartHandlePayload(byte[] data)
throws IOException {
@@ -201,7 +206,7 @@ public class S3AMultipartUploader extends MultipartUploader {
}
final long len = input.readLong();
final String etag = input.readUTF();
- if (len <= 0) {
+ if (len < 0) {
throw new IOException("Negative length");
}
return Pair.of(len, etag);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
index d28f39b..059312a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.fs.contract.s3a;
-import java.io.IOException;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -26,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractMultipartUploaderTest;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.WriteOperationHelper;
@@ -35,6 +34,9 @@ import static org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles.DEFAULT_H
/**
* Test MultipartUploader with S3A.
+ * Although not an S3A Scale test subclass, it uses the -Dscale option
+ * to enable it, and partition size option to control the size of
+ * parts uploaded.
*/
public class ITestS3AContractMultipartUploader extends
AbstractContractMultipartUploaderTest {
@@ -79,6 +81,35 @@ public class ITestS3AContractMultipartUploader extends
return new S3AContract(conf);
}
+ /**
+ * Bigger test: use the scale timeout.
+ * @return the timeout for scale tests.
+ */
+ @Override
+ protected int getTestTimeoutMillis() {
+ return SCALE_TEST_TIMEOUT_MILLIS;
+ }
+
+
+ @Override
+ protected boolean supportsConcurrentUploadsToSamePath() {
+ return true;
+ }
+
+ /**
+ * Provide a pessimistic time to become consistent.
+ * @return a time in milliseconds
+ */
+ @Override
+ protected int timeToBecomeConsistentMillis() {
+ return 30 * 1000;
+ }
+
+ @Override
+ protected boolean finalizeConsumesUploadIdImmediately() {
+ return false;
+ }
+
@Override
public void setup() throws Exception {
super.setup();
@@ -103,14 +134,29 @@ public class ITestS3AContractMultipartUploader extends
public void teardown() throws Exception {
Path teardown = path("teardown").getParent();
S3AFileSystem fs = getFileSystem();
- WriteOperationHelper helper = fs.getWriteOperationHelper();
- try {
- LOG.info("Teardown: aborting outstanding uploads under {}", teardown);
- int count = helper.abortMultipartUploadsUnderPath(fs.pathToKey(teardown));
- LOG.info("Found {} incomplete uploads", count);
- } catch (IOException e) {
- LOG.warn("IOE in teardown", e);
+ if (fs != null) {
+ WriteOperationHelper helper = fs.getWriteOperationHelper();
+ try {
+ LOG.info("Teardown: aborting outstanding uploads under {}", teardown);
+ int count = helper.abortMultipartUploadsUnderPath(
+ fs.pathToKey(teardown));
+ LOG.info("Found {} incomplete uploads", count);
+ } catch (Exception e) {
+ LOG.warn("Exeception in teardown", e);
+ }
}
super.teardown();
}
+
+ /**
+ * S3 has no concept of directories, so this test does not apply.
+ */
+ public void testDirectoryInTheWay() throws Exception {
+ // no-op
+ }
+
+ @Override
+ public void testMultipartUploadReverseOrder() throws Exception {
+ ContractTestUtils.skip("skipped for speed");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1d24f84/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
index 35d0460..4825d26 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AMultipartUploaderSupport.java
@@ -60,7 +60,7 @@ public class TestS3AMultipartUploaderSupport extends HadoopTestBase {
@Test
public void testNoLen() throws Throwable {
intercept(IllegalArgumentException.class,
- () -> buildPartHandlePayload("tag", 0));
+ () -> buildPartHandlePayload("tag", -1));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HADOOP-15970. Upgrade plexus-utils
from 2.0.5 to 3.1.0.
Posted by sh...@apache.org.
HADOOP-15970. Upgrade plexus-utils from 2.0.5 to 3.1.0.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff31313d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff31313d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff31313d
Branch: refs/heads/HDFS-12943
Commit: ff31313d83ae26be5e29eaad7d9ce6184bae83e2
Parents: a8bbd81
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Dec 4 16:13:50 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Dec 4 22:45:29 2018 +0900
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff31313d/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a205d2b..5fd3a56 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -716,7 +716,7 @@
<dependency>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
- <version>2.0.5</version>
+ <version>3.1.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.plexus</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HADOOP-15957. WASB: Add asterisk
wildcard support for PageBlobDirSet.
Posted by sh...@apache.org.
HADOOP-15957. WASB: Add asterisk wildcard support for PageBlobDirSet.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ccb640a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ccb640a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ccb640a
Branch: refs/heads/HDFS-12943
Commit: 7ccb640a66bb5bb9f657a2db85bcc8ae0ded4892
Parents: c9bfca2
Author: Da Zhou <da...@microsoft.com>
Authored: Fri Nov 30 10:12:41 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 30 10:12:41 2018 +0000
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 62 ++++++-
.../azure/ITestNativeAzureFileSystemLive.java | 4 +-
.../fs/azure/TestKeyPageBlobDirectories.java | 170 +++++++++++++++++++
3 files changed, 231 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ccb640a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index d2f9ca6..7bf3420 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -44,6 +44,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobContainerWrapper;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
@@ -241,6 +242,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private static final String HTTP_SCHEME = "http";
private static final String HTTPS_SCHEME = "https";
private static final String WASB_AUTHORITY_DELIMITER = "@";
+ private static final char ASTERISK_SYMBOL = '*';
private static final String AZURE_ROOT_CONTAINER = "$root";
private static final int DEFAULT_CONCURRENT_WRITES = 8;
@@ -1169,7 +1171,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
for (String currentDir : rawDirs) {
String myDir;
try {
- myDir = verifyAndConvertToStandardFormat(currentDir);
+ myDir = verifyAndConvertToStandardFormat(currentDir.trim());
} catch (URISyntaxException ex) {
throw new AzureException(String.format(
"The directory %s specified in the configuration entry %s is not"
@@ -1214,7 +1216,12 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
public boolean isKeyForDirectorySet(String key, Set<String> dirSet) {
String defaultFS = FileSystem.getDefaultUri(sessionConfiguration).toString();
for (String dir : dirSet) {
- if (dir.isEmpty() || key.startsWith(dir + "/")) {
+ if (dir.isEmpty()) {
+ // dir is root
+ return true;
+ }
+
+ if (matchAsteriskPattern(key, dir)) {
return true;
}
@@ -1227,7 +1234,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Concatenate the default file system prefix with the relative
// page blob directory path.
//
- if (key.startsWith(trim(defaultFS, "/") + "/" + dir + "/")){
+ String dirWithPrefix = trim(defaultFS, "/") + "/" + dir;
+ if (matchAsteriskPattern(key, dirWithPrefix)) {
return true;
}
}
@@ -1238,6 +1246,54 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
return false;
}
+ private boolean matchAsteriskPattern(String pathName, String pattern) {
+ if (pathName == null || pathName.length() == 0) {
+ return false;
+ }
+
+ int pathIndex = 0;
+ int patternIndex = 0;
+
+ while (pathIndex < pathName.length() && patternIndex < pattern.length()) {
+ char charToMatch = pattern.charAt(patternIndex);
+
+ // normal char:
+ if (charToMatch != ASTERISK_SYMBOL) {
+ if (charToMatch != pathName.charAt(pathIndex)) {
+ return false;
+ }
+ pathIndex++;
+ patternIndex++;
+ continue;
+ }
+
+ // ASTERISK_SYMBOL
+ // 1. * is used in path name: *a/b,a*/b, a/*b, a/b*
+ if (patternIndex > 0 && pattern.charAt(patternIndex - 1) != Path.SEPARATOR_CHAR
+ || patternIndex + 1 < pattern.length() && pattern.charAt(patternIndex + 1) != Path.SEPARATOR_CHAR) {
+ if (ASTERISK_SYMBOL != pathName.charAt(pathIndex)) {
+ return false;
+ }
+
+ pathIndex++;
+ patternIndex++;
+ continue;
+ }
+
+ // 2. * is used as wildcard: */a, a/*/b, a/*
+ patternIndex++;
+ // find next path separator
+ while (pathIndex < pathName.length() && pathName.charAt(pathIndex) != Path.SEPARATOR_CHAR) {
+ pathIndex++;
+ }
+ }
+
+ // Ensure it is not a file/dir which shares same prefix as pattern
+ // Eg: pattern: /A/B, pathName: /A/BBB should not match
+ return patternIndex == pattern.length()
+ && (pathIndex == pathName.length() || pathName.charAt(pathIndex) == Path.SEPARATOR_CHAR);
+ }
+
/**
* Returns the file block size. This is a fake value used for integration
* of the Azure store with Hadoop.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ccb640a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
index 9033674..f86af95 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemLive.java
@@ -230,7 +230,7 @@ public class ITestNativeAzureFileSystemLive extends
// negative tests
String[] negativeKeys = { "", "/", "bar", "bar/", "bar/pageBlobs", "bar/pageBlobs/foo",
- "bar/pageBlobs/foo/", "/pageBlobs/", "/pageBlobs", "pageBlobs", "pageBlobsxyz/" };
+ "bar/pageBlobs/foo/", "/pageBlobs/", "/pageBlobs", "pageBlobsxyz/" };
for (String s : negativeKeys) {
assertFalse(store.isPageBlobKey(s));
assertFalse(store.isPageBlobKey(uriPrefix + s));
@@ -262,7 +262,7 @@ public class ITestNativeAzureFileSystemLive extends
// negative tests
String[] negativeKeys = { "", "/", "bar", "bar/", "bar/hbase",
- "bar/hbase/foo", "bar/hbase/foo/", "/hbase/", "/hbase", "hbase",
+ "bar/hbase/foo", "bar/hbase/foo/", "/hbase/", "/hbase",
"hbasexyz/", "foo/atomicRenameDir1/"};
for (String s : negativeKeys) {
assertFalse(store.isAtomicRenameKey(s));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ccb640a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java
new file mode 100644
index 0000000..7cc4878
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestKeyPageBlobDirectories.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.Test;
+
+/**
+ * Test config property KEY_PAGE_BLOB_DIRECTORIES.
+ */
+public class TestKeyPageBlobDirectories extends AbstractWasbTestBase{
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.create();
+ }
+
+ public void expectPageBlobKey(boolean expectedOutcome, AzureNativeFileSystemStore store, String path) {
+ assertEquals("Unexpected result for isPageBlobKey(" + path + ")",
+ expectedOutcome, store.isPageBlobKey(path));
+
+ }
+
+ @Test
+ public void testKeySetWithoutAsterisk() throws Exception {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ String dirList = "/service/WALs,/data/mypageblobfiles";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/dir/recovered.edits");
+ expectPageBlobKey(true, store, "service/WALs/recovered.edits");
+
+ expectPageBlobKey(false, store, "data/dir/recovered.txt");
+ expectPageBlobKey(true, store, "data/mypageblobfiles/recovered.txt");
+ }
+
+ @Test
+ public void testKeySetWithAsterisk() throws Exception {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ String dirList = "/service/*/*/*/recovered.edits,/*/recovered.edits,/*/*/*/WALs, /*/*/oldWALs/*/*";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/dir/recovered.edits");
+ expectPageBlobKey(true, store, "service/dir1/dir2/dir3/recovered.edits");
+
+ expectPageBlobKey(false, store, "data/dir/recovered.edits");
+ expectPageBlobKey(true, store, "data/recovered.edits");
+
+ expectPageBlobKey(false, store, "dir1/dir2/WALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs/data1");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs/data2");
+
+ expectPageBlobKey(false, store, "dir1/oldWALs/data");
+ expectPageBlobKey(false, store, "dir1/dir2/oldWALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/oldWALs/dir3/dir4/data");
+ }
+
+
+
+ @Test
+ public void testKeySetUsingFullName() throws Exception {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ String dirList = "/service/WALs,/data/mypageblobfiles,/*/*/WALs,/*/*/recover.edits";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ final String defaultFS = FileSystem.getDefaultUri(conf).toString();
+
+ expectPageBlobKey(false, store, defaultFS + "service/recover.edits");
+ expectPageBlobKey(true, store, defaultFS + "service/WALs/recover.edits");
+
+ expectPageBlobKey(false, store, defaultFS + "data/mismatch/mypageblobfiles/data");
+ expectPageBlobKey(true, store, defaultFS + "data/mypageblobfiles/data");
+
+ expectPageBlobKey(false, store, defaultFS + "dir1/dir2/dir3/WALs/data");
+ expectPageBlobKey(true, store, defaultFS + "dir1/dir2/WALs/data");
+
+ expectPageBlobKey(false, store, defaultFS + "dir1/dir2/dir3/recover.edits");
+ expectPageBlobKey(true, store, defaultFS + "dir1/dir2/recover.edits");
+
+ }
+
+ @Test
+ public void testKeyContainsAsterisk() throws IOException {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ // Test dir name which contains *
+ String dirList = "/service/*/*/*/d*ir,/*/fi**le.data,/*/*/*/WALs*, /*/*/oldWALs";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/d*ir/data");
+ expectPageBlobKey(true, store, "service/dir1/dir2/dir3/d*ir/data");
+
+ expectPageBlobKey(false, store, "dir/fi*le.data");
+ expectPageBlobKey(true, store, "dir/fi**le.data");
+
+ expectPageBlobKey(false, store, "dir1/dir2/WALs/data");
+ expectPageBlobKey(false, store, "dir1/dir2/dir3/WALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs*/data1");
+ expectPageBlobKey(true, store, "dir1/dir2/dir3/WALs*/data2");
+
+ expectPageBlobKey(false, store, "dir1/oldWALs/data");
+ expectPageBlobKey(true, store, "dir1/dir2/oldWALs/data1");
+ expectPageBlobKey(true, store, "dir1/dir2/oldWALs/data2");
+ }
+
+ @Test
+ public void testKeyWithCommonPrefix() throws IOException {
+ NativeAzureFileSystem azureFs = fs;
+ AzureNativeFileSystemStore store = azureFs.getStore();
+ Configuration conf = fs.getConf();
+ // Test dir name which contains *
+ String dirList = "/service/WALs,/*/*/WALs";
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, dirList);
+ URI uri = fs.getUri();
+ fs.initialize(uri, conf);
+
+ expectPageBlobKey(false, store, "/");
+ expectPageBlobKey(false, store, "service");
+
+ expectPageBlobKey(false, store, "service/WALsssssss/dir");
+ expectPageBlobKey(true, store, "service/WALs/dir");
+
+ expectPageBlobKey(false, store, "service/dir/WALsss/data");
+ expectPageBlobKey(true, store, "service/dir/WALs/data");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-9069. Fix
SchedulerInfo#getSchedulerType for custom schedulers. Contributed by Bilwa S
T.
Posted by sh...@apache.org.
YARN-9069. Fix SchedulerInfo#getSchedulerType for custom schedulers. Contributed by Bilwa S T.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07142f54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07142f54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07142f54
Branch: refs/heads/HDFS-12943
Commit: 07142f54a8c7f70857e99c041f3a2a5189c809b5
Parents: a68d766
Author: bibinchundatt <bi...@apache.org>
Authored: Thu Nov 29 22:02:59 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Thu Nov 29 22:02:59 2018 +0530
----------------------------------------------------------------------
.../yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07142f54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 163f707..ede0d15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -54,6 +54,8 @@ public class SchedulerInfo {
this.schedulerName = "Fair Scheduler";
} else if (rs instanceof FifoScheduler) {
this.schedulerName = "Fifo Scheduler";
+ } else {
+ this.schedulerName = rs.getClass().getSimpleName();
}
this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org