You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/10/02 21:40:01 UTC
[01/50] [abbrv] hadoop git commit: HADOOP-15723. ABFS: Ranger
Support. Contributed by Yuan Gao.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-13532 d6ee4290d -> e8b860431
HADOOP-15723. ABFS: Ranger Support.
Contributed by Yuan Gao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5da9928
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5da9928
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5da9928
Branch: refs/heads/HDFS-13532
Commit: d5da9928c99073abdd27b66f794fe4312434ff96
Parents: 26c94a0
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 25 19:12:21 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 25 19:13:10 2018 +0100
----------------------------------------------------------------------
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 36 ++
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 116 ++++++-
.../azurebfs/constants/ConfigurationKeys.java | 2 +
.../extensions/AbfsAuthorizationException.java | 41 +++
.../fs/azurebfs/extensions/AbfsAuthorizer.java | 57 +++
.../azurebfs/AbstractAbfsIntegrationTest.java | 8 +-
.../ITestAzureBlobFileSystemAuthorization.java | 346 +++++++++++++++++++
.../ITestAbfsFileSystemContractDistCp.java | 3 +
.../azurebfs/extensions/MockAbfsAuthorizer.java | 87 +++++
.../fs/azurebfs/extensions/package-info.java | 22 ++
10 files changed, 698 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index f0088ff..c57c340 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.azurebfs;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
@@ -42,6 +43,8 @@ import org.apache.hadoop.fs.azurebfs.diagnostics.BooleanConfigurationBasicValida
import org.apache.hadoop.fs.azurebfs.diagnostics.IntegerConfigurationBasicValidator;
import org.apache.hadoop.fs.azurebfs.diagnostics.LongConfigurationBasicValidator;
import org.apache.hadoop.fs.azurebfs.diagnostics.StringConfigurationBasicValidator;
+import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizationException;
+import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizer;
import org.apache.hadoop.fs.azurebfs.extensions.CustomTokenProviderAdaptee;
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider;
@@ -155,6 +158,10 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_ENABLE_DELEGATION_TOKEN)
private boolean enableDelegationToken;
+ @StringConfigurationValidatorAnnotation(ConfigurationKey = ABFS_EXTERNAL_AUTHORIZATION_CLASS,
+ DefaultValue = "")
+ private String abfsExternalAuthorizationClass;
+
private Map<String, String> storageAccountKeys;
public AbfsConfiguration(final Configuration rawConfig, String accountName)
@@ -490,6 +497,35 @@ public class AbfsConfiguration{
}
}
+ public String getAbfsExternalAuthorizationClass() {
+ return this.abfsExternalAuthorizationClass;
+ }
+
+ public AbfsAuthorizer getAbfsAuthorizer() throws IOException {
+ String authClassName = getAbfsExternalAuthorizationClass();
+ AbfsAuthorizer authorizer = null;
+
+ try {
+ if (authClassName != null && !authClassName.isEmpty()) {
+ @SuppressWarnings("unchecked")
+ Class<AbfsAuthorizer> authClass = (Class<AbfsAuthorizer>) rawConfig.getClassByName(authClassName);
+ authorizer = authClass.getConstructor(new Class[] {Configuration.class}).newInstance(rawConfig);
+ authorizer.init();
+ }
+ } catch (
+ IllegalAccessException
+ | InstantiationException
+ | ClassNotFoundException
+ | IllegalArgumentException
+ | InvocationTargetException
+ | NoSuchMethodException
+ | SecurityException
+ | AbfsAuthorizationException e) {
+ throw new IOException(e);
+ }
+ return authorizer;
+ }
+
void validateStorageAccountKeys() throws InvalidConfigurationValueException {
Base64StringConfigurationBasicValidator validator = new Base64StringConfigurationBasicValidator(
FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, "", true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 200f3e7..4b521e1 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -28,6 +28,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -62,9 +63,12 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.FileSystemOperationUnh
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriAuthorityException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidUriException;
import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
+import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizationException;
+import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizer;
import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
@@ -87,6 +91,7 @@ public class AzureBlobFileSystem extends FileSystem {
private boolean delegationTokenEnabled = false;
private AbfsDelegationTokenManager delegationTokenManager;
+ private AbfsAuthorizer authorizer;
@Override
public void initialize(URI uri, Configuration configuration)
@@ -132,6 +137,10 @@ public class AzureBlobFileSystem extends FileSystem {
}
AbfsClientThrottlingIntercept.initializeSingleton(abfsConfiguration.isAutoThrottlingEnabled());
+
+ // Initialize ABFS authorizer
+ //
+ this.authorizer = abfsConfiguration.getAbfsAuthorizer();
}
@Override
@@ -158,8 +167,11 @@ public class AzureBlobFileSystem extends FileSystem {
public FSDataInputStream open(final Path path, final int bufferSize) throws IOException {
LOG.debug("AzureBlobFileSystem.open path: {} bufferSize: {}", path, bufferSize);
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.READ, qualifiedPath);
+
try {
- InputStream inputStream = abfsStore.openFileForRead(makeQualified(path), statistics);
+ InputStream inputStream = abfsStore.openFileForRead(qualifiedPath, statistics);
return new FSDataInputStream(inputStream);
} catch(AzureBlobFileSystemException ex) {
checkException(path, ex);
@@ -176,8 +188,11 @@ public class AzureBlobFileSystem extends FileSystem {
overwrite,
blockSize);
+ Path qualifiedPath = makeQualified(f);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- OutputStream outputStream = abfsStore.createFile(makeQualified(f), overwrite,
+ OutputStream outputStream = abfsStore.createFile(qualifiedPath, overwrite,
permission == null ? FsPermission.getFileDefault() : permission, FsPermission.getUMask(getConf()));
return new FSDataOutputStream(outputStream, statistics);
} catch(AzureBlobFileSystemException ex) {
@@ -236,8 +251,11 @@ public class AzureBlobFileSystem extends FileSystem {
f.toString(),
bufferSize);
+ Path qualifiedPath = makeQualified(f);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- OutputStream outputStream = abfsStore.openFileForWrite(makeQualified(f), false);
+ OutputStream outputStream = abfsStore.openFileForWrite(qualifiedPath, false);
return new FSDataOutputStream(outputStream, statistics);
} catch(AzureBlobFileSystemException ex) {
checkException(f, ex);
@@ -267,7 +285,11 @@ public class AzureBlobFileSystem extends FileSystem {
adjustedDst = new Path(dst, sourceFileName);
}
- abfsStore.rename(makeQualified(src), makeQualified(adjustedDst));
+ Path qualifiedSrcPath = makeQualified(src);
+ Path qualifiedDstPath = makeQualified(adjustedDst);
+ performAbfsAuthCheck(FsAction.READ_WRITE, qualifiedSrcPath, qualifiedDstPath);
+
+ abfsStore.rename(qualifiedSrcPath, qualifiedDstPath);
return true;
} catch(AzureBlobFileSystemException ex) {
checkException(
@@ -289,6 +311,9 @@ public class AzureBlobFileSystem extends FileSystem {
LOG.debug(
"AzureBlobFileSystem.delete path: {} recursive: {}", f.toString(), recursive);
+ Path qualifiedPath = makeQualified(f);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
if (f.isRoot()) {
if (!recursive) {
return false;
@@ -298,7 +323,7 @@ public class AzureBlobFileSystem extends FileSystem {
}
try {
- abfsStore.delete(makeQualified(f), recursive);
+ abfsStore.delete(qualifiedPath, recursive);
return true;
} catch (AzureBlobFileSystemException ex) {
checkException(f, ex, AzureServiceErrorCode.PATH_NOT_FOUND);
@@ -312,8 +337,11 @@ public class AzureBlobFileSystem extends FileSystem {
LOG.debug(
"AzureBlobFileSystem.listStatus path: {}", f.toString());
+ Path qualifiedPath = makeQualified(f);
+ performAbfsAuthCheck(FsAction.READ, qualifiedPath);
+
try {
- FileStatus[] result = abfsStore.listStatus(makeQualified(f));
+ FileStatus[] result = abfsStore.listStatus(qualifiedPath);
return result;
} catch (AzureBlobFileSystemException ex) {
checkException(f, ex);
@@ -332,8 +360,11 @@ public class AzureBlobFileSystem extends FileSystem {
return true;
}
+ Path qualifiedPath = makeQualified(f);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.createDirectory(makeQualified(f), permission == null ? FsPermission.getDirDefault() : permission,
+ abfsStore.createDirectory(qualifiedPath, permission == null ? FsPermission.getDirDefault() : permission,
FsPermission.getUMask(getConf()));
return true;
} catch (AzureBlobFileSystemException ex) {
@@ -357,8 +388,11 @@ public class AzureBlobFileSystem extends FileSystem {
public FileStatus getFileStatus(final Path f) throws IOException {
LOG.debug("AzureBlobFileSystem.getFileStatus path: {}", f);
+ Path qualifiedPath = makeQualified(f);
+ performAbfsAuthCheck(FsAction.READ, qualifiedPath);
+
try {
- return abfsStore.getFileStatus(makeQualified(f));
+ return abfsStore.getFileStatus(qualifiedPath);
} catch(AzureBlobFileSystemException ex) {
checkException(f, ex);
return null;
@@ -528,8 +562,11 @@ public class AzureBlobFileSystem extends FileSystem {
throw new IllegalArgumentException("A valid owner or group must be specified.");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.setOwner(makeQualified(path),
+ abfsStore.setOwner(qualifiedPath,
owner,
group);
} catch (AzureBlobFileSystemException ex) {
@@ -556,8 +593,11 @@ public class AzureBlobFileSystem extends FileSystem {
throw new IllegalArgumentException("The permission can't be null");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.setPermission(makeQualified(path),
+ abfsStore.setPermission(qualifiedPath,
permission);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
@@ -589,8 +629,11 @@ public class AzureBlobFileSystem extends FileSystem {
throw new IllegalArgumentException("The value of the aclSpec parameter is invalid.");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.modifyAclEntries(makeQualified(path),
+ abfsStore.modifyAclEntries(qualifiedPath,
aclSpec);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
@@ -620,8 +663,11 @@ public class AzureBlobFileSystem extends FileSystem {
throw new IllegalArgumentException("The aclSpec argument is invalid.");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.removeAclEntries(makeQualified(path), aclSpec);
+ abfsStore.removeAclEntries(qualifiedPath, aclSpec);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
}
@@ -643,8 +689,11 @@ public class AzureBlobFileSystem extends FileSystem {
+ "hierarchical namespace enabled.");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.removeDefaultAcl(makeQualified(path));
+ abfsStore.removeDefaultAcl(qualifiedPath);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
}
@@ -668,8 +717,11 @@ public class AzureBlobFileSystem extends FileSystem {
+ "hierarchical namespace enabled.");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.removeAcl(makeQualified(path));
+ abfsStore.removeAcl(qualifiedPath);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
}
@@ -700,8 +752,11 @@ public class AzureBlobFileSystem extends FileSystem {
throw new IllegalArgumentException("The aclSpec argument is invalid.");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.WRITE, qualifiedPath);
+
try {
- abfsStore.setAcl(makeQualified(path), aclSpec);
+ abfsStore.setAcl(qualifiedPath, aclSpec);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
}
@@ -724,8 +779,11 @@ public class AzureBlobFileSystem extends FileSystem {
+ "hierarchical namespace enabled.");
}
+ Path qualifiedPath = makeQualified(path);
+ performAbfsAuthCheck(FsAction.READ, qualifiedPath);
+
try {
- return abfsStore.getAclStatus(makeQualified(path));
+ return abfsStore.getAclStatus(qualifiedPath);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
return null;
@@ -950,4 +1008,30 @@ public class AzureBlobFileSystem extends FileSystem {
boolean getIsNamespaceEnabeld() throws AzureBlobFileSystemException {
return abfsStore.getIsNamespaceEnabled();
}
+
+ /**
+ * Use ABFS authorizer to check if user is authorized to perform specific
+ * {@link FsAction} on specified {@link Path}s.
+ *
+ * @param action The {@link FsAction} being requested on the provided {@link Path}s.
+ * @param paths The absolute paths of the storage being accessed.
+ * @throws AbfsAuthorizationException on authorization failure.
+ * @throws IOException network problems or similar.
+ * @throws IllegalArgumentException if the required parameters are not provided.
+ */
+ private void performAbfsAuthCheck(FsAction action, Path... paths)
+ throws AbfsAuthorizationException, IOException {
+ if (authorizer == null) {
+ LOG.debug("ABFS authorizer is not initialized. No authorization check will be performed.");
+ } else {
+ Preconditions.checkArgument(paths.length > 0, "no paths supplied for authorization check");
+
+ LOG.debug("Auth check for action: {} on paths: {}", action.toString(), Arrays.toString(paths));
+ if (!authorizer.isAuthorized(action, paths)) {
+ throw new AbfsAuthorizationException(
+ "User is not authorized for action " + action.toString()
+ + " on paths: " + Arrays.toString(paths));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 13cdaeb..3e4ae21 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -85,5 +85,7 @@ public final class ConfigurationKeys {
public static final String FS_AZURE_ENABLE_DELEGATION_TOKEN = "fs.azure.enable.delegation.token";
public static final String FS_AZURE_DELEGATION_TOKEN_PROVIDER_TYPE = "fs.azure.delegation.token.provider.type";
+ public static final String ABFS_EXTERNAL_AUTHORIZATION_CLASS = "abfs.external.authorization.class";
+
private ConfigurationKeys() {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizationException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizationException.java
new file mode 100644
index 0000000..64a4820
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizationException.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.extensions;
+
+import java.io.IOException;
+
+/**
+ * Exception raised on ABFS Authorization failures.
+ */
+public class AbfsAuthorizationException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ public AbfsAuthorizationException(String message, Exception e) {
+ super(message, e);
+ }
+
+ public AbfsAuthorizationException(String message) {
+ super(message);
+ }
+
+ public AbfsAuthorizationException(Throwable e) {
+ super(e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizer.java
new file mode 100644
index 0000000..f4495ec
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/AbfsAuthorizer.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.extensions;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+
+/**
+ * Interface to support authorization in Azure Blob File System.
+ */
+@InterfaceAudience.LimitedPrivate("authorization-subsystems")
+@InterfaceStability.Unstable
+public interface AbfsAuthorizer {
+
+ /**
+ * Initialize authorizer for Azure Blob File System.
+ *
+ * @throws AbfsAuthorizationException if unable to initialize the authorizer.
+ * @throws IOException network problems or similar.
+ * @throws IllegalArgumentException if the required parameters are not provided.
+ */
+ void init() throws AbfsAuthorizationException, IOException;
+
+ /**
+ * Checks if the provided {@link FsAction} is allowed on the provided {@link Path}s.
+ *
+ * @param action the {@link FsAction} being requested on the provided {@link Path}s.
+ * @param absolutePaths The absolute paths of the storage being accessed.
+ * @return true if authorized, otherwise false.
+ * @throws AbfsAuthorizationException on authorization failure.
+ * @throws IOException network problems or similar.
+ * @throws IllegalArgumentException if the required parameters are not provided.
+ */
+ boolean isAuthorized(FsAction action, Path... absolutePaths)
+ throws AbfsAuthorizationException, IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index 52185cd..6f794d0 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -257,12 +257,12 @@ public abstract class AbstractAbfsIntegrationTest extends
return abfsConfig.getRawConfiguration();
}
- protected boolean isIPAddress() {
- return isIPAddress;
+ public AuthType getAuthType() {
+ return this.authType;
}
- protected AuthType getAuthType() {
- return this.authType;
+ protected boolean isIPAddress() {
+ return isIPAddress;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
new file mode 100644
index 0000000..37ab825
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
@@ -0,0 +1,346 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
+import org.apache.hadoop.fs.azurebfs.extensions.AbfsAuthorizationException;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+
+import static org.apache.hadoop.fs.azurebfs.extensions.MockAbfsAuthorizer.*;
+import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry;
+import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
+import static org.apache.hadoop.fs.permission.AclEntryType.GROUP;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test Perform Authorization Check operation
+ */
+public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrationTest {
+
+ private static final Path TEST_READ_ONLY_FILE_PATH_0 = new Path(TEST_READ_ONLY_FILE_0);
+ private static final Path TEST_READ_ONLY_FOLDER_PATH = new Path(TEST_READ_ONLY_FOLDER);
+ private static final Path TEST_WRITE_ONLY_FILE_PATH_0 = new Path(TEST_WRITE_ONLY_FILE_0);
+ private static final Path TEST_WRITE_ONLY_FILE_PATH_1 = new Path(TEST_WRITE_ONLY_FILE_1);
+ private static final Path TEST_READ_WRITE_FILE_PATH_0 = new Path(TEST_READ_WRITE_FILE_0);
+ private static final Path TEST_READ_WRITE_FILE_PATH_1 = new Path(TEST_READ_WRITE_FILE_1);
+ private static final Path TEST_WRITE_ONLY_FOLDER_PATH = new Path(TEST_WRITE_ONLY_FOLDER);
+ private static final Path TEST_WRITE_THEN_READ_ONLY_PATH = new Path(TEST_WRITE_THEN_READ_ONLY);
+ private static final String TEST_AUTHZ_CLASS = "org.apache.hadoop.fs.azurebfs.extensions.MockAbfsAuthorizer";
+
+ public ITestAzureBlobFileSystemAuthorization() throws Exception {
+ }
+
+ @Override
+ public void setup() throws Exception {
+ this.getConfiguration().set(ConfigurationKeys.ABFS_EXTERNAL_AUTHORIZATION_CLASS, TEST_AUTHZ_CLASS);
+ super.setup();
+ }
+
+ @Test
+ public void testOpenFileWithInvalidPath() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ intercept(IllegalArgumentException.class,
+ ()-> {
+ fs.open(new Path("")).close();
+ });
+ }
+
+ @Test
+ public void testOpenFileAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ fs.open(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ }
+
+ @Test
+ public void testOpenFileUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.open(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ });
+ }
+
+ @Test
+ public void testCreateFileAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ }
+
+ @Test
+ public void testCreateFileUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.create(TEST_READ_ONLY_FILE_PATH_0).close();
+ });
+ }
+
+ @Test
+ public void testAppendFileAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ fs.append(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ }
+
+ @Test
+ public void testAppendFileUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.append(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ });
+ }
+
+ @Test
+ public void testRenameAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.rename(TEST_READ_WRITE_FILE_PATH_0, TEST_READ_WRITE_FILE_PATH_1);
+ }
+
+ @Test
+ public void testRenameUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.rename(TEST_WRITE_ONLY_FILE_PATH_0, TEST_WRITE_ONLY_FILE_PATH_1);
+ });
+ }
+
+ @Test
+ public void testDeleteFileAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ fs.delete(TEST_WRITE_ONLY_FILE_PATH_0, false);
+ }
+
+ @Test
+ public void testDeleteFileUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = this.getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.delete(TEST_WRITE_THEN_READ_ONLY_PATH, false);
+ });
+ }
+
+ @Test
+ public void testListStatusAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ fs.listStatus(TEST_WRITE_THEN_READ_ONLY_PATH);
+ }
+
+ @Test
+ public void testListStatusUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.listStatus(TEST_WRITE_ONLY_FILE_PATH_0);
+ });
+ }
+
+ @Test
+ public void testMkDirsAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(TEST_WRITE_ONLY_FOLDER_PATH, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ }
+
+ @Test
+ public void testMkDirsUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.mkdirs(TEST_READ_ONLY_FOLDER_PATH, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ });
+ }
+
+ @Test
+ public void testGetFileStatusAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ fs.getFileStatus(TEST_WRITE_THEN_READ_ONLY_PATH);
+ }
+
+ @Test
+ public void testGetFileStatusUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.getFileStatus(TEST_WRITE_ONLY_FILE_PATH_0);
+ });
+ }
+
+ @Test
+ public void testSetOwnerAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ fs.setOwner(TEST_WRITE_ONLY_FILE_PATH_0, "testUser", "testGroup");
+ }
+
+ @Test
+ public void testSetOwnerUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.setOwner(TEST_WRITE_THEN_READ_ONLY_PATH, "testUser", "testGroup");
+ });
+ }
+
+ @Test
+ public void testSetPermissionAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ fs.setPermission(TEST_WRITE_ONLY_FILE_PATH_0, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ }
+
+ @Test
+ public void testSetPermissionUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.setPermission(TEST_WRITE_THEN_READ_ONLY_PATH, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ });
+ }
+
+ @Test
+ public void testModifyAclEntriesAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ fs.modifyAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
+ }
+
+ @Test
+ public void testModifyAclEntriesUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.modifyAclEntries(TEST_WRITE_THEN_READ_ONLY_PATH, aclSpec);
+ });
+ }
+
+ @Test
+ public void testRemoveAclEntriesAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ //fs.modifyAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
+ fs.removeAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
+ }
+
+ @Test
+ public void testRemoveAclEntriesUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.removeAclEntries(TEST_WRITE_THEN_READ_ONLY_PATH, aclSpec);
+ });
+ }
+
+ @Test
+ public void testRemoveDefaultAclAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ fs.removeDefaultAcl(TEST_WRITE_ONLY_FILE_PATH_0);
+ }
+
+ @Test
+ public void testRemoveDefaultAclUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.removeDefaultAcl(TEST_WRITE_THEN_READ_ONLY_PATH);
+ });
+ }
+
+ @Test
+ public void testRemoveAclAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ fs.removeAcl(TEST_WRITE_ONLY_FILE_PATH_0);
+ }
+
+ @Test
+ public void testRemoveAclUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.removeAcl(TEST_WRITE_THEN_READ_ONLY_PATH);
+ });
+ }
+
+ @Test
+ public void testSetAclAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ fs.setAcl(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
+ }
+
+ @Test
+ public void testSetAclUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.setAcl(TEST_WRITE_THEN_READ_ONLY_PATH, aclSpec);
+ });
+ }
+
+ @Test
+ public void testGetAclStatusAuthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ fs.getAclStatus(TEST_WRITE_THEN_READ_ONLY_PATH);
+ }
+
+ @Test
+ public void testGetAclStatusUnauthorized() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
+ List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
+ intercept(AbfsAuthorizationException.class,
+ ()-> {
+ fs.getAclStatus(TEST_WRITE_ONLY_FILE_PATH_0);
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java
index 529fe83..0c7db73 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractDistCp.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.fs.azurebfs.contract;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+import org.junit.Assume;
/**
* Contract test for distCp operation.
@@ -29,6 +31,7 @@ public class ITestAbfsFileSystemContractDistCp extends AbstractContractDistCpTes
public ITestAbfsFileSystemContractDistCp() throws Exception {
binding = new ABFSContractTestBinding();
+ Assume.assumeTrue(binding.getAuthType() != AuthType.OAuth);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockAbfsAuthorizer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockAbfsAuthorizer.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockAbfsAuthorizer.java
new file mode 100644
index 0000000..6820edd
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockAbfsAuthorizer.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.extensions;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+
+/**
+ * A mock Azure Blob File System Authorization Implementation
+ */
+public class MockAbfsAuthorizer implements AbfsAuthorizer {
+
+ public static final String TEST_READ_ONLY_FILE_0 = "readOnlyFile0";
+ public static final String TEST_READ_ONLY_FILE_1 = "readOnlyFile1";
+ public static final String TEST_READ_ONLY_FOLDER = "readOnlyFolder";
+ public static final String TEST_WRITE_ONLY_FILE_0 = "writeOnlyFile0";
+ public static final String TEST_WRITE_ONLY_FILE_1 = "writeOnlyFile1";
+ public static final String TEST_WRITE_ONLY_FOLDER = "writeOnlyFolder";
+ public static final String TEST_READ_WRITE_FILE_0 = "readWriteFile0";
+ public static final String TEST_READ_WRITE_FILE_1 = "readWriteFile1";
+ public static final String TEST_WRITE_THEN_READ_ONLY = "writeThenReadOnlyFile";
+ private Configuration conf;
+ private Set<Path> readOnlyPaths = new HashSet<Path>();
+ private Set<Path> writeOnlyPaths = new HashSet<Path>();
+ private Set<Path> readWritePaths = new HashSet<Path>();
+ private int writeThenReadOnly = 0;
+ public MockAbfsAuthorizer(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public void init() throws AbfsAuthorizationException, IOException {
+ readOnlyPaths.add(new Path(TEST_READ_ONLY_FILE_0));
+ readOnlyPaths.add(new Path(TEST_READ_ONLY_FILE_1));
+ readOnlyPaths.add(new Path(TEST_READ_ONLY_FOLDER));
+ writeOnlyPaths.add(new Path(TEST_WRITE_ONLY_FILE_0));
+ writeOnlyPaths.add(new Path(TEST_WRITE_ONLY_FILE_1));
+ writeOnlyPaths.add(new Path(TEST_WRITE_ONLY_FOLDER));
+ readWritePaths.add(new Path(TEST_READ_WRITE_FILE_0));
+ readWritePaths.add(new Path(TEST_READ_WRITE_FILE_1));
+ }
+
+ @Override
+ public boolean isAuthorized(FsAction action, Path... absolutePaths) throws AbfsAuthorizationException, IOException {
+ Set<Path> paths = new HashSet<Path>();
+ for (Path path : absolutePaths) {
+ paths.add(new Path(path.getName()));
+ }
+
+ if (action.equals(FsAction.READ) && Stream.concat(readOnlyPaths.stream(), readWritePaths.stream()).collect(Collectors.toSet()).containsAll(paths)) {
+ return true;
+ } else if (action.equals(FsAction.READ) && paths.contains(new Path(TEST_WRITE_THEN_READ_ONLY)) && writeThenReadOnly == 1) {
+ return true;
+ } else if (action.equals(FsAction.WRITE)
+ && Stream.concat(writeOnlyPaths.stream(), readWritePaths.stream()).collect(Collectors.toSet()).containsAll(paths)) {
+ return true;
+ } else if (action.equals(FsAction.WRITE) && paths.contains(new Path(TEST_WRITE_THEN_READ_ONLY)) && writeThenReadOnly == 0) {
+ writeThenReadOnly = 1;
+ return true;
+ } else {
+ return action.equals(FsAction.READ_WRITE) && readWritePaths.containsAll(paths);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5da9928/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java
new file mode 100644
index 0000000..48a468b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.azurebfs.extensions;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-8468. Enable the use of queue based maximum container allocation limit and implement it in FairScheduler. Contributed by Antal Bálint Steinbach.
Posted by in...@apache.org.
YARN-8468. Enable the use of queue based maximum container allocation limit and implement it in FairScheduler. Contributed by Antal Bálint Steinbach.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd6be589
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd6be589
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd6be589
Branch: refs/heads/HDFS-13532
Commit: fd6be5898ad1a650e3bceacb8169a53520da57e5
Parents: 0da03f8
Author: Weiwei Yang <ww...@apache.org>
Authored: Sat Sep 29 17:47:12 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Sat Sep 29 17:47:12 2018 +0800
----------------------------------------------------------------------
.../server/resourcemanager/RMAppManager.java | 11 +-
.../server/resourcemanager/RMServerUtils.java | 9 +-
.../scheduler/AbstractYarnScheduler.java | 21 ++-
.../scheduler/SchedulerUtils.java | 62 +++----
.../scheduler/YarnScheduler.java | 9 +-
.../scheduler/capacity/CapacityScheduler.java | 7 +-
.../processor/PlacementConstraintProcessor.java | 12 +-
.../scheduler/fair/AllocationConfiguration.java | 13 ++
.../scheduler/fair/FSLeafQueue.java | 10 ++
.../scheduler/fair/FSParentQueue.java | 15 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 7 +
.../scheduler/fair/FairScheduler.java | 52 ++++--
.../allocation/AllocationFileQueueParser.java | 7 +
.../fair/allocation/QueueProperties.java | 16 +-
.../webapp/FairSchedulerPage.java | 4 +
.../webapp/dao/FairSchedulerQueueInfo.java | 9 +-
.../resourcemanager/AppManagerTestBase.java | 107 +++++++++++
.../yarn/server/resourcemanager/MockRM.java | 8 +
.../server/resourcemanager/TestAppManager.java | 89 ++--------
.../TestAppManagerWithFairScheduler.java | 177 +++++++++++++++++++
.../resourcemanager/TestClientRMService.java | 104 +++++------
.../resourcemanager/TestRMServerUtils.java | 98 +++++++++-
.../scheduler/TestSchedulerUtils.java | 144 ++++++++-------
.../capacity/TestCapacityScheduler.java | 39 +++-
.../scheduler/fair/FairSchedulerTestBase.java | 4 +-
.../fair/TestAllocationFileLoaderService.java | 37 +++-
.../TestApplicationMasterServiceWithFS.java | 174 ++++++++++++++++++
.../scheduler/fair/TestFairScheduler.java | 140 +++++++++++++--
.../allocationfile/AllocationFileQueue.java | 6 +-
.../AllocationFileQueueBuilder.java | 6 +
.../AllocationFileQueueProperties.java | 12 ++
.../src/site/markdown/FairScheduler.md | 2 +
32 files changed, 1115 insertions(+), 296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index d0f2ce6..783fab0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -567,13 +568,13 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// Normalize all requests
String queue = submissionContext.getQueue();
+ Resource maxAllocation = scheduler.getMaximumResourceCapability(queue);
for (ResourceRequest amReq : amReqs) {
- SchedulerUtils.normalizeAndValidateRequest(amReq,
- scheduler.getMaximumResourceCapability(queue),
- queue, scheduler, isRecovery, rmContext);
+ SchedulerUtils.normalizeAndValidateRequest(amReq, maxAllocation,
+ queue, scheduler, isRecovery, rmContext, null);
- amReq.setCapability(
- scheduler.getNormalizedResource(amReq.getCapability()));
+ amReq.setCapability(scheduler.getNormalizedResource(
+ amReq.getCapability(), maxAllocation));
}
return amReqs;
} catch (InvalidResourceRequestException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index ab6bbcf..b18b12e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -97,7 +97,7 @@ public class RMServerUtils {
"INCORRECT_CONTAINER_VERSION_ERROR";
private static final String INVALID_CONTAINER_ID =
"INVALID_CONTAINER_ID";
- private static final String RESOURCE_OUTSIDE_ALLOWED_RANGE =
+ public static final String RESOURCE_OUTSIDE_ALLOWED_RANGE =
"RESOURCE_OUTSIDE_ALLOWED_RANGE";
protected static final RecordFactory RECORD_FACTORY =
@@ -235,7 +235,7 @@ public class RMServerUtils {
* requested memory/vcore is non-negative and not greater than max
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
- Resource maximumResource, String queueName, YarnScheduler scheduler,
+ Resource maximumAllocation, String queueName, YarnScheduler scheduler,
RMContext rmContext) throws InvalidResourceRequestException {
// Get queue from scheduler
QueueInfo queueInfo = null;
@@ -247,7 +247,7 @@ public class RMServerUtils {
}
for (ResourceRequest resReq : ask) {
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
+ SchedulerUtils.normalizeAndValidateRequest(resReq, maximumAllocation,
queueName, scheduler, rmContext, queueInfo);
}
}
@@ -338,7 +338,8 @@ public class RMServerUtils {
return false;
}
ResourceScheduler scheduler = rmContext.getScheduler();
- request.setCapability(scheduler.getNormalizedResource(request.getCapability()));
+ request.setCapability(scheduler
+ .getNormalizedResource(request.getCapability(), maximumAllocation));
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 9d2b058..0acfca7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -1159,11 +1159,12 @@ public abstract class AbstractYarnScheduler
}
@Override
- public Resource getNormalizedResource(Resource requestedResource) {
+ public Resource getNormalizedResource(Resource requestedResource,
+ Resource maxResourceCapability) {
return SchedulerUtils.getNormalizedResource(requestedResource,
getResourceCalculator(),
getMinimumResourceCapability(),
- getMaximumResourceCapability(),
+ maxResourceCapability,
getMinimumResourceCapability());
}
@@ -1173,8 +1174,20 @@ public abstract class AbstractYarnScheduler
* @param asks resource requests
*/
protected void normalizeResourceRequests(List<ResourceRequest> asks) {
- for (ResourceRequest ask: asks) {
- ask.setCapability(getNormalizedResource(ask.getCapability()));
+ normalizeResourceRequests(asks, null);
+ }
+
+ /**
+ * Normalize a list of resource requests
+ * using queue maximum resource allocations.
+ * @param asks resource requests
+ */
+ protected void normalizeResourceRequests(List<ResourceRequest> asks,
+ String queueName) {
+ Resource maxAllocation = getMaximumResourceCapability(queueName);
+ for (ResourceRequest ask : asks) {
+ ask.setCapability(
+ getNormalizedResource(ask.getCapability(), maxAllocation));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 9b07d37..9a02b6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -114,19 +114,19 @@ public class SchedulerUtils {
public static final String UPDATED_CONTAINER =
"Temporary container killed by application for ExeutionType update";
-
- public static final String LOST_CONTAINER =
+
+ public static final String LOST_CONTAINER =
"Container released on a *lost* node";
-
- public static final String PREEMPTED_CONTAINER =
+
+ public static final String PREEMPTED_CONTAINER =
"Container preempted by scheduler";
-
- public static final String COMPLETED_APPLICATION =
+
+ public static final String COMPLETED_APPLICATION =
"Container of a completed application";
-
+
public static final String EXPIRED_CONTAINER =
"Container expired since it was unused";
-
+
public static final String UNRESERVED_CONTAINER =
"Container reservation no longer required.";
@@ -141,7 +141,7 @@ public class SchedulerUtils {
*/
public static ContainerStatus createAbnormalContainerStatus(
ContainerId containerId, String diagnostics) {
- return createAbnormalContainerStatus(containerId,
+ return createAbnormalContainerStatus(containerId,
ContainerExitStatus.ABORTED, diagnostics);
}
@@ -169,14 +169,14 @@ public class SchedulerUtils {
*/
public static ContainerStatus createPreemptedContainerStatus(
ContainerId containerId, String diagnostics) {
- return createAbnormalContainerStatus(containerId,
+ return createAbnormalContainerStatus(containerId,
ContainerExitStatus.PREEMPTED, diagnostics);
}
/**
* Utility to create a {@link ContainerStatus} during exceptional
* circumstances.
- *
+ *
* @param containerId {@link ContainerId} of returned/released/lost container.
* @param diagnostics diagnostic message
* @return <code>ContainerStatus</code> for an returned/released/lost
@@ -184,7 +184,7 @@ public class SchedulerUtils {
*/
private static ContainerStatus createAbnormalContainerStatus(
ContainerId containerId, int exitStatus, String diagnostics) {
- ContainerStatus containerStatus =
+ ContainerStatus containerStatus =
recordFactory.newRecordInstance(ContainerStatus.class);
containerStatus.setContainerId(containerId);
containerStatus.setDiagnostics(diagnostics);
@@ -254,23 +254,14 @@ public class SchedulerUtils {
labelExp = RMNodeLabelsManager.NO_LABEL;
}
- if ( labelExp != null) {
+ if (labelExp != null) {
resReq.setNodeLabelExpression(labelExp);
}
}
public static void normalizeAndValidateRequest(ResourceRequest resReq,
- Resource maximumResource, String queueName, YarnScheduler scheduler,
- boolean isRecovery, RMContext rmContext)
- throws InvalidResourceRequestException {
- normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
- isRecovery, rmContext, null);
- }
-
-
- private static void normalizeAndValidateRequest(ResourceRequest resReq,
- Resource maximumResource, String queueName, YarnScheduler scheduler,
- boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
+ Resource maximumAllocation, String queueName, YarnScheduler scheduler,
+ boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
throws InvalidResourceRequestException {
Configuration conf = rmContext.getYarnConfiguration();
// If Node label is not enabled throw exception
@@ -299,37 +290,30 @@ public class SchedulerUtils {
SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
if (!isRecovery) {
- validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
+ validateResourceRequest(resReq, maximumAllocation, queueInfo, rmContext);
}
}
- public static void normalizeAndvalidateRequest(ResourceRequest resReq,
- Resource maximumResource, String queueName, YarnScheduler scheduler,
- RMContext rmContext) throws InvalidResourceRequestException {
- normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler,
- rmContext, null);
- }
-
- public static void normalizeAndvalidateRequest(ResourceRequest resReq,
- Resource maximumResource, String queueName, YarnScheduler scheduler,
+ public static void normalizeAndValidateRequest(ResourceRequest resReq,
+ Resource maximumAllocation, String queueName, YarnScheduler scheduler,
RMContext rmContext, QueueInfo queueInfo)
throws InvalidResourceRequestException {
- normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
+ normalizeAndValidateRequest(resReq, maximumAllocation, queueName, scheduler,
false, rmContext, queueInfo);
}
/**
* Utility method to validate a resource request, by insuring that the
* requested memory/vcore is non-negative and not greater than max
- *
+ *
* @throws InvalidResourceRequestException when there is invalid request
*/
private static void validateResourceRequest(ResourceRequest resReq,
- Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
+ Resource maximumAllocation, QueueInfo queueInfo, RMContext rmContext)
throws InvalidResourceRequestException {
final Resource requestedResource = resReq.getCapability();
checkResourceRequestAgainstAvailableResource(requestedResource,
- maximumResource);
+ maximumAllocation);
String labelExp = resReq.getNodeLabelExpression();
// we don't allow specify label expression other than resourceName=ANY now
@@ -535,7 +519,7 @@ public class SchedulerUtils {
if (!str.trim().isEmpty()) {
// check queue label
if (queueLabels == null) {
- return false;
+ return false;
} else {
if (!queueLabels.contains(str)
&& !queueLabels.contains(RMNodeLabelsManager.ANY)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 0f7a5b5..d95fe7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -390,12 +390,17 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
SchedulerNode getSchedulerNode(NodeId nodeId);
/**
- * Normalize a resource request.
+ * Normalize a resource request using scheduler level maximum resource or
+ * queue based maximum resource.
*
* @param requestedResource the resource to be normalized
+ * @param maxResourceCapability Maximum container allocation value, if null or
+ * empty scheduler level maximum container allocation value will be
+ * used
* @return the normalized resource
*/
- Resource getNormalizedResource(Resource requestedResource);
+ Resource getNormalizedResource(Resource requestedResource,
+ Resource maxResourceCapability);
/**
* Verify whether a submitted application lifetime is valid as per configured
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 4b274df..75d6144 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1159,10 +1159,12 @@ public class CapacityScheduler extends
if (asks == null) {
return;
}
+ Resource maxAllocation = getMaximumResourceCapability();
for (SchedulingRequest ask: asks) {
ResourceSizing sizing = ask.getResourceSizing();
if (sizing != null && sizing.getResources() != null) {
- sizing.setResources(getNormalizedResource(sizing.getResources()));
+ sizing.setResources(
+ getNormalizedResource(sizing.getResources(), maxAllocation));
}
}
}
@@ -2567,6 +2569,9 @@ public class CapacityScheduler extends
@Override
public Resource getMaximumResourceCapability(String queueName) {
+ if(queueName == null || queueName.isEmpty()) {
+ return getMaximumResourceCapability();
+ }
CSQueue queue = getQueue(queueName);
if (queue == null) {
LOG.error("Unknown queue: " + queueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
index cf944a6..c4c9574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java
@@ -175,11 +175,19 @@ public class PlacementConstraintProcessor extends AbstractPlacementProcessor {
private void dispatchRequestsForPlacement(ApplicationAttemptId appAttemptId,
List<SchedulingRequest> schedulingRequests) {
if (schedulingRequests != null && !schedulingRequests.isEmpty()) {
+ SchedulerApplicationAttempt appAttempt =
+ scheduler.getApplicationAttempt(appAttemptId);
+ String queueName = null;
+ if(appAttempt != null) {
+ queueName = appAttempt.getQueueName();
+ }
+ Resource maxAllocation =
+ scheduler.getMaximumResourceCapability(queueName);
// Normalize the Requests before dispatching
schedulingRequests.forEach(req -> {
Resource reqResource = req.getResourceSizing().getResources();
- req.getResourceSizing()
- .setResources(this.scheduler.getNormalizedResource(reqResource));
+ req.getResourceSizing().setResources(
+ this.scheduler.getNormalizedResource(reqResource, maxAllocation));
});
this.placementDispatcher.dispatch(new BatchedRequests(iteratorType,
appAttemptId.getApplicationId(), schedulingRequests, 1));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index e48e04b..826d9f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -91,6 +91,9 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
private final SchedulingPolicy defaultSchedulingPolicy;
+ //Map for maximum container resource allocation per queues by queue name
+ private final Map<String, Resource> queueMaxContainerAllocationMap;
+
// Policy for mapping apps to queues
@VisibleForTesting
QueuePlacementPolicy placementPolicy;
@@ -138,6 +141,8 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
this.placementPolicy = newPlacementPolicy;
this.configuredQueues = queueProperties.getConfiguredQueues();
this.nonPreemptableQueues = queueProperties.getNonPreemptableQueues();
+ this.queueMaxContainerAllocationMap =
+ queueProperties.getMaxContainerAllocation();
}
public AllocationConfiguration(Configuration conf) {
@@ -167,6 +172,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
placementPolicy =
QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
nonPreemptableQueues = new HashSet<>();
+ queueMaxContainerAllocationMap = new HashMap<>();
}
/**
@@ -272,6 +278,12 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return maxQueueResource;
}
+ @VisibleForTesting
+ Resource getQueueMaxContainerAllocation(String queue) {
+ Resource resource = queueMaxContainerAllocationMap.get(queue);
+ return resource == null ? Resources.unbounded() : resource;
+ }
+
/**
* Get the maximum resource allocation for children of the given queue.
*
@@ -375,6 +387,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
queue.setMaxRunningApps(getQueueMaxApps(name));
queue.setMaxAMShare(getQueueMaxAMShare(name));
queue.setMaxChildQueueResource(getMaxChildResources(name));
+ queue.setMaxContainerAllocation(getQueueMaxContainerAllocation(name));
// Set queue metrics.
queue.getMetrics().setMinShare(queue.getMinShare());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index cbc74d2..7e4dab8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -547,6 +547,16 @@ public class FSLeafQueue extends FSQueue {
this.weights = weight;
}
+ @Override
+ public Resource getMaximumContainerAllocation() {
+ if (maxContainerAllocation.equals(Resources.unbounded())
+ && getParent() != null) {
+ return getParent().getMaximumContainerAllocation();
+ } else {
+ return maxContainerAllocation;
+ }
+ }
+
/**
* Helper method to compute the amount of minshare starvation.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index d5df549..e9f4af6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -59,7 +59,20 @@ public class FSParentQueue extends FSQueue {
FSParentQueue parent) {
super(name, scheduler, parent);
}
-
+
+ @Override
+ public Resource getMaximumContainerAllocation() {
+ if (getName().equals("root")) {
+ return maxContainerAllocation;
+ }
+ if (maxContainerAllocation.equals(Resources.unbounded())
+ && getParent() != null) {
+ return getParent().getMaximumContainerAllocation();
+ } else {
+ return maxContainerAllocation;
+ }
+ }
+
void addChildQueue(FSQueue child) {
writeLock.lock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 6b88a32..6217f55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -84,6 +84,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private float fairSharePreemptionThreshold = 0.5f;
private boolean preemptable = true;
private boolean isDynamic = true;
+ protected Resource maxContainerAllocation;
public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) {
this.name = name;
@@ -163,6 +164,12 @@ public abstract class FSQueue implements Queue, Schedulable {
this.maxShare = maxShare;
}
+ public void setMaxContainerAllocation(Resource maxContainerAllocation){
+ this.maxContainerAllocation = maxContainerAllocation;
+ }
+
+ public abstract Resource getMaximumContainerAllocation();
+
@Override
public Resource getMaxShare() {
Resource maxResource = maxShare.getResource(scheduler.getClusterResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 43a47ae..da5e4c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -192,6 +192,7 @@ public class FairScheduler extends
protected long rackLocalityDelayMs; // Delay for rack locality
protected boolean assignMultiple; // Allocate multiple containers per
// heartbeat
+
@VisibleForTesting
boolean maxAssignDynamic;
protected int maxAssign; // Max containers to assign per heartbeat
@@ -227,12 +228,12 @@ public class FairScheduler extends
private void validateConf(FairSchedulerConfiguration config) {
// validate scheduler memory allocation setting
- int minMem = config.getInt(
- YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- int maxMem = config.getInt(
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ int minMem =
+ config.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ int maxMem =
+ config.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
if (minMem < 0 || minMem > maxMem) {
throw new YarnRuntimeException("Invalid resource scheduler memory"
@@ -254,12 +255,12 @@ public class FairScheduler extends
}
// validate scheduler vcores allocation setting
- int minVcores = config.getInt(
- YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
- int maxVcores = config.getInt(
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ int minVcores =
+ config.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ int maxVcores =
+ config.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
if (minVcores < 0 || minVcores > maxVcores) {
throw new YarnRuntimeException("Invalid resource scheduler vcores"
@@ -833,14 +834,35 @@ public class FairScheduler extends
}
@Override
- public Resource getNormalizedResource(Resource requestedResource) {
+ public Resource getNormalizedResource(Resource requestedResource,
+ Resource maxResourceCapability) {
return SchedulerUtils.getNormalizedResource(requestedResource,
DOMINANT_RESOURCE_CALCULATOR,
minimumAllocation,
- getMaximumResourceCapability(),
+ maxResourceCapability,
incrAllocation);
}
+ @Override
+ public Resource getMaximumResourceCapability(String queueName) {
+ if(queueName == null || queueName.isEmpty()) {
+ return getMaximumResourceCapability();
+ }
+ FSQueue queue = queueMgr.getQueue(queueName);
+ Resource schedulerLevelMaxResourceCapability =
+ getMaximumResourceCapability();
+ if (queue == null) {
+ return schedulerLevelMaxResourceCapability;
+ }
+ Resource queueMaxResourceCapability = queue.getMaximumContainerAllocation();
+ if (queueMaxResourceCapability.equals(Resources.unbounded())) {
+ return schedulerLevelMaxResourceCapability;
+ } else {
+ return Resources.componentwiseMin(schedulerLevelMaxResourceCapability,
+ queueMaxResourceCapability);
+ }
+ }
+
@VisibleForTesting
@Override
public void killContainer(RMContainer container) {
@@ -897,7 +919,7 @@ public class FairScheduler extends
handleContainerUpdates(application, updateRequests);
// Sanity check
- normalizeResourceRequests(ask);
+ normalizeResourceRequests(ask, queue.getName());
// TODO, normalize SchedulingRequest
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
index 441c34a..72c6c68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
@@ -51,6 +51,8 @@ public class AllocationFileQueueParser {
private static final String MAX_CHILD_RESOURCES = "maxChildResources";
private static final String MAX_RUNNING_APPS = "maxRunningApps";
private static final String MAX_AMSHARE = "maxAMShare";
+ public static final String MAX_CONTAINER_ALLOCATION =
+ "maxContainerAllocation";
private static final String WEIGHT = "weight";
private static final String MIN_SHARE_PREEMPTION_TIMEOUT =
"minSharePreemptionTimeout";
@@ -155,6 +157,11 @@ public class AllocationFileQueueParser {
float val = Float.parseFloat(text);
val = Math.min(val, 1.0f);
builder.queueMaxAMShares(queueName, val);
+ } else if (MAX_CONTAINER_ALLOCATION.equals(field.getTagName())) {
+ String text = getTrimmedTextData(field);
+ ConfigurableResource val =
+ FairSchedulerConfiguration.parseResourceConfigValue(text);
+ builder.queueMaxContainerAllocation(queueName, val.getResource());
} else if (WEIGHT.equals(field.getTagName())) {
String text = getTrimmedTextData(field);
double val = Double.parseDouble(text);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
index ee5f179..badf05f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
@@ -53,6 +53,7 @@ public class QueueProperties {
private final Set<String> reservableQueues;
private final Set<String> nonPreemptableQueues;
private final Map<FSQueueType, Set<String>> configuredQueues;
+ private final Map<String, Resource> queueMaxContainerAllocation;
QueueProperties(Builder builder) {
this.reservableQueues = builder.reservableQueues;
@@ -70,6 +71,7 @@ public class QueueProperties {
this.maxChildQueueResources = builder.maxChildQueueResources;
this.reservationAcls = builder.reservationAcls;
this.queueAcls = builder.queueAcls;
+ this.queueMaxContainerAllocation = builder.queueMaxContainerAllocation;
}
public Map<FSQueueType, Set<String>> getConfiguredQueues() {
@@ -133,7 +135,11 @@ public class QueueProperties {
return nonPreemptableQueues;
}
- /**
+ public Map<String, Resource> getMaxContainerAllocation() {
+ return queueMaxContainerAllocation;
+ }
+
+ /**
* Builder class for {@link QueueProperties}.
* All methods are adding queue properties to the maps of this builder
* keyed by the queue's name except some methods
@@ -149,6 +155,7 @@ public class QueueProperties {
new HashMap<>();
private Map<String, Integer> queueMaxApps = new HashMap<>();
private Map<String, Float> queueMaxAMShares = new HashMap<>();
+ private Map<String, Resource> queueMaxContainerAllocation = new HashMap<>();
private Map<String, Float> queueWeights = new HashMap<>();
private Map<String, SchedulingPolicy> queuePolicies = new HashMap<>();
private Map<String, Long> minSharePreemptionTimeouts = new HashMap<>();
@@ -253,6 +260,12 @@ public class QueueProperties {
return this;
}
+ public Builder queueMaxContainerAllocation(String queueName,
+ Resource value) {
+ queueMaxContainerAllocation.put(queueName, value);
+ return this;
+ }
+
public void configuredQueues(FSQueueType queueType, String queueName) {
this.configuredQueues.get(queueType).add(queueName);
}
@@ -275,6 +288,5 @@ public class QueueProperties {
public QueueProperties build() {
return new QueueProperties(this);
}
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
index ef417d4..7f31def 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
@@ -78,6 +78,8 @@ public class FairSchedulerPage extends RmView {
__("Num Pending Applications:", qinfo.getNumPendingApplications()).
__("Min Resources:", qinfo.getMinResources().toString()).
__("Max Resources:", qinfo.getMaxResources().toString()).
+ __("Max Container Allocation:",
+ qinfo.getMaxContainerAllocation().toString()).
__("Reserved Resources:", qinfo.getReservedResources().toString());
int maxApps = qinfo.getMaxApplications();
if (maxApps < Integer.MAX_VALUE) {
@@ -107,6 +109,8 @@ public class FairSchedulerPage extends RmView {
__("Used Resources:", qinfo.getUsedResources().toString()).
__("Min Resources:", qinfo.getMinResources().toString()).
__("Max Resources:", qinfo.getMaxResources().toString()).
+ __("Max Container Allocation:",
+ qinfo.getMaxContainerAllocation().toString()).
__("Reserved Resources:", qinfo.getReservedResources().toString());
int maxApps = qinfo.getMaxApplications();
if (maxApps < Integer.MAX_VALUE) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 913513c..70c5fd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -60,6 +60,7 @@ public class FairSchedulerQueueInfo {
private ResourceInfo fairResources;
private ResourceInfo clusterResources;
private ResourceInfo reservedResources;
+ private ResourceInfo maxContainerAllocation;
private long allocatedContainers;
private long reservedContainers;
@@ -99,6 +100,8 @@ public class FairSchedulerQueueInfo {
maxResources = new ResourceInfo(
Resources.componentwiseMin(queue.getMaxShare(),
scheduler.getClusterResource()));
+ maxContainerAllocation =
+ new ResourceInfo(scheduler.getMaximumResourceCapability(queueName));
reservedResources = new ResourceInfo(queue.getReservedResource());
fractionMemSteadyFairShare =
@@ -186,7 +189,11 @@ public class FairSchedulerQueueInfo {
public ResourceInfo getMaxResources() {
return maxResources;
}
-
+
+ public ResourceInfo getMaxContainerAllocation() {
+ return maxContainerAllocation;
+ }
+
public ResourceInfo getReservedResources() {
return reservedResources;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
new file mode 100644
index 0000000..36258b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import static java.util.stream.Collectors.toSet;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.mockito.ArgumentCaptor;
+
+/**
+ * Base class for AppManager related test.
+ *
+ */
+public class AppManagerTestBase {
+
+ // Extend and make the functions we want to test public
+ protected class TestRMAppManager extends RMAppManager {
+ private final RMStateStore stateStore;
+
+ public TestRMAppManager(RMContext context, Configuration conf) {
+ super(context, null, null, new ApplicationACLsManager(conf), conf);
+ this.stateStore = context.getStateStore();
+ }
+
+ public TestRMAppManager(RMContext context,
+ ClientToAMTokenSecretManagerInRM clientToAMSecretManager,
+ YarnScheduler scheduler, ApplicationMasterService masterService,
+ ApplicationACLsManager applicationACLsManager, Configuration conf) {
+ super(context, scheduler, masterService, applicationACLsManager, conf);
+ this.stateStore = context.getStateStore();
+ }
+
+ public void checkAppNumCompletedLimit() {
+ super.checkAppNumCompletedLimit();
+ }
+
+ public void finishApplication(ApplicationId appId) {
+ super.finishApplication(appId);
+ }
+
+ public int getCompletedAppsListSize() {
+ return super.getCompletedAppsListSize();
+ }
+
+ public int getNumberOfCompletedAppsInStateStore() {
+ return this.completedAppsInStateStore;
+ }
+
+ public List<ApplicationId> getCompletedApps() {
+ return completedApps;
+ }
+
+ public Set<ApplicationId> getFirstNCompletedApps(int n) {
+ return getCompletedApps().stream().limit(n).collect(toSet());
+ }
+
+ public Set<ApplicationId> getCompletedAppsWithEvenIdsInRange(int n) {
+ return getCompletedApps().stream().limit(n)
+ .filter(app -> app.getId() % 2 == 0).collect(toSet());
+ }
+
+ public Set<ApplicationId> getRemovedAppsFromStateStore(int numRemoves) {
+ ArgumentCaptor<RMApp> argumentCaptor =
+ ArgumentCaptor.forClass(RMApp.class);
+ verify(stateStore, times(numRemoves))
+ .removeApplication(argumentCaptor.capture());
+ return argumentCaptor.getAllValues().stream().map(RMApp::getApplicationId)
+ .collect(toSet());
+ }
+
+ public void submitApplication(
+ ApplicationSubmissionContext submissionContext, String user)
+ throws YarnException {
+ super.submitApplication(submissionContext, System.currentTimeMillis(),
+ user);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index df86f28..408cb18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -513,6 +513,14 @@ public class MockRM extends ResourceManager {
return submitApp(masterMemory, false);
}
+ public RMApp submitApp(int masterMemory, String queue) throws Exception {
+ return submitApp(masterMemory, "",
+ UserGroupInformation.getCurrentUser().getShortUserName(), null, false,
+ queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS),
+ null);
+ }
+
public RMApp submitApp(int masterMemory, Set<String> appTags)
throws Exception {
Resource resource = Resource.newInstance(masterMemory, 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 0bd5372..8f48293 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessM
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
@@ -82,12 +81,10 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.ArgumentCaptor;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.io.IOException;
-import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -99,10 +96,10 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
-import static java.util.stream.Collectors.toSet;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.isA;
import static org.mockito.Matchers.matches;
import static org.mockito.Mockito.doAnswer;
@@ -117,7 +114,7 @@ import static org.mockito.Mockito.when;
*
*/
-public class TestAppManager{
+public class TestAppManager extends AppManagerTestBase{
private Log LOG = LogFactory.getLog(TestAppManager.class);
private static RMAppEventType appEventType = RMAppEventType.KILL;
@@ -234,70 +231,6 @@ public class TestAppManager{
setAppEventType(event.getType());
System.out.println("in handle routine " + getAppEventType().toString());
}
- }
-
-
- // Extend and make the functions we want to test public
- public class TestRMAppManager extends RMAppManager {
- private final RMStateStore stateStore;
-
- public TestRMAppManager(RMContext context, Configuration conf) {
- super(context, null, null, new ApplicationACLsManager(conf), conf);
- this.stateStore = context.getStateStore();
- }
-
- public TestRMAppManager(RMContext context,
- ClientToAMTokenSecretManagerInRM clientToAMSecretManager,
- YarnScheduler scheduler, ApplicationMasterService masterService,
- ApplicationACLsManager applicationACLsManager, Configuration conf) {
- super(context, scheduler, masterService, applicationACLsManager, conf);
- this.stateStore = context.getStateStore();
- }
-
- public void checkAppNumCompletedLimit() {
- super.checkAppNumCompletedLimit();
- }
-
- public void finishApplication(ApplicationId appId) {
- super.finishApplication(appId);
- }
-
- public int getCompletedAppsListSize() {
- return super.getCompletedAppsListSize();
- }
-
- public int getNumberOfCompletedAppsInStateStore() {
- return this.completedAppsInStateStore;
- }
-
- List<ApplicationId> getCompletedApps() {
- return completedApps;
- }
-
- Set<ApplicationId> getFirstNCompletedApps(int n) {
- return getCompletedApps().stream().limit(n).collect(toSet());
- }
-
- Set<ApplicationId> getCompletedAppsWithEvenIdsInRange(int n) {
- return getCompletedApps().stream().limit(n)
- .filter(app -> app.getId() % 2 == 0).collect(toSet());
- }
-
- Set<ApplicationId> getRemovedAppsFromStateStore(int numRemoves) {
- ArgumentCaptor<RMApp> argumentCaptor =
- ArgumentCaptor.forClass(RMApp.class);
- verify(stateStore, times(numRemoves))
- .removeApplication(argumentCaptor.capture());
- return argumentCaptor.getAllValues().stream().map(RMApp::getApplicationId)
- .collect(toSet());
- }
-
- public void submitApplication(
- ApplicationSubmissionContext submissionContext, String user)
- throws YarnException, IOException {
- super.submitApplication(submissionContext, System.currentTimeMillis(),
- user);
- }
}
private void addToCompletedApps(TestRMAppManager appMonitor,
@@ -1213,17 +1146,21 @@ public class TestAppManager{
Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+ when(scheduler.getMaximumResourceCapability(anyString())).thenReturn(
+ Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+
ResourceCalculator rs = mock(ResourceCalculator.class);
when(scheduler.getResourceCalculator()).thenReturn(rs);
- when(scheduler.getNormalizedResource(any()))
+ when(scheduler.getNormalizedResource(any(), any()))
.thenAnswer(new Answer<Resource>() {
- @Override
- public Resource answer(InvocationOnMock invocationOnMock)
- throws Throwable {
- return (Resource) invocationOnMock.getArguments()[0];
- }
- });
+ @Override
+ public Resource answer(InvocationOnMock invocationOnMock)
+ throws Throwable {
+ return (Resource) invocationOnMock.getArguments()[0];
+ }
+ });
return scheduler;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java
new file mode 100644
index 0000000..feb7ed2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import static org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException.InvalidResourceType.GREATER_THEN_MAX_ALLOCATION;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.matches;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashMap;
+
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Testing applications being retired from RM with fair scheduler.
+ *
+ */
+public class TestAppManagerWithFairScheduler extends AppManagerTestBase{
+
+ private static final String TEST_FOLDER = "test-queues";
+
+ private static YarnConfiguration conf = new YarnConfiguration();
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ String allocFile =
+ GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath();
+
+ int queueMaxAllocation = 512;
+
+ PrintWriter out = new PrintWriter(new FileWriter(allocFile));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\">");
+ out.println(" <maxContainerAllocation>" + queueMaxAllocation
+ + " mb 1 vcores" + "</maxContainerAllocation>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueB\">");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+ ResourceScheduler.class);
+
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
+ }
+
+ @AfterClass
+ public static void teardown(){
+ File allocFile = GenericTestUtils.getTestDir(TEST_FOLDER);
+ allocFile.delete();
+ }
+
+ @Test
+ public void testQueueSubmitWithHighQueueContainerSize()
+ throws YarnException {
+
+ ApplicationId appId = MockApps.newAppID(1);
+ RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+
+ Resource resource = Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+
+ ApplicationSubmissionContext asContext =
+ recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
+ asContext.setApplicationId(appId);
+ asContext.setResource(resource);
+ asContext.setPriority(Priority.newInstance(0));
+ asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
+ asContext.setQueue("queueA");
+ QueueInfo mockDefaultQueueInfo = mock(QueueInfo.class);
+
+ // Setup a PlacementManager returns a new queue
+ PlacementManager placementMgr = mock(PlacementManager.class);
+ doAnswer(new Answer<ApplicationPlacementContext>() {
+
+ @Override
+ public ApplicationPlacementContext answer(InvocationOnMock invocation)
+ throws Throwable {
+ return new ApplicationPlacementContext("queueA");
+ }
+
+ }).when(placementMgr).placeApplication(
+ any(ApplicationSubmissionContext.class), matches("test1"));
+ doAnswer(new Answer<ApplicationPlacementContext>() {
+
+ @Override
+ public ApplicationPlacementContext answer(InvocationOnMock invocation)
+ throws Throwable {
+ return new ApplicationPlacementContext("queueB");
+ }
+
+ }).when(placementMgr).placeApplication(
+ any(ApplicationSubmissionContext.class), matches("test2"));
+
+ MockRM newMockRM = new MockRM(conf);
+ RMContext newMockRMContext = newMockRM.getRMContext();
+ newMockRMContext.setQueuePlacementManager(placementMgr);
+ ApplicationMasterService masterService = new ApplicationMasterService(
+ newMockRMContext, newMockRMContext.getScheduler());
+
+ TestRMAppManager newAppMonitor = new TestRMAppManager(newMockRMContext,
+ new ClientToAMTokenSecretManagerInRM(), newMockRMContext.getScheduler(),
+ masterService, new ApplicationACLsManager(conf), conf);
+
+ // only user test has permission to submit to 'test' queue
+
+ try {
+ newAppMonitor.submitApplication(asContext, "test1");
+ Assert.fail("Test should fail on too high allocation!");
+ } catch (InvalidResourceRequestException e) {
+ Assert.assertEquals(GREATER_THEN_MAX_ALLOCATION,
+ e.getInvalidResourceType());
+ }
+
+ // Should not throw exception
+ newAppMonitor.submitApplication(asContext, "test2");
+ }
+
+ private static ContainerLaunchContext mockContainerLaunchContext(
+ RecordFactory recordFactory) {
+ ContainerLaunchContext amContainer = recordFactory.newRecordInstance(
+ ContainerLaunchContext.class);
+ amContainer
+ .setApplicationACLs(new HashMap<ApplicationAccessType, String>());
+ return amContainer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 61048a5..3fe46b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -362,9 +362,9 @@ public class TestClientRMService {
@Test
public void testGetApplicationReport() throws Exception {
- YarnScheduler yarnScheduler = mock(YarnScheduler.class);
+ ResourceScheduler scheduler = mock(ResourceScheduler.class);
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
ApplicationId appId1 = getApplicationId(1);
@@ -373,7 +373,7 @@ public class TestClientRMService {
mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
- ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
+ ClientRMService rmService = new ClientRMService(rmContext, scheduler,
null, mockAclsManager, null, null);
try {
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
@@ -456,9 +456,9 @@ public class TestClientRMService {
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
IOException {
ApplicationAttemptId attemptId = getApplicationAttemptId(1);
- YarnScheduler yarnScheduler = mockYarnScheduler();
+ ResourceScheduler scheduler = mockResourceScheduler();
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
new EventHandler<Event>() {
public void handle(Event event) {
@@ -468,7 +468,7 @@ public class TestClientRMService {
mock(ApplicationSubmissionContext.class);
YarnConfiguration config = new YarnConfiguration();
RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
- rmContext, yarnScheduler, null, asContext, config, null, null);
+ rmContext, scheduler, null, asContext, config, null, null);
ApplicationResourceUsageReport report = rmAppAttemptImpl
.getApplicationResourceUsageReport();
assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
@@ -537,14 +537,14 @@ public class TestClientRMService {
}
public ClientRMService createRMService() throws IOException, YarnException {
- YarnScheduler yarnScheduler = mockYarnScheduler();
+ ResourceScheduler scheduler = mockResourceScheduler();
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
- yarnScheduler);
+ scheduler);
when(rmContext.getRMApps()).thenReturn(apps);
when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
- RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
+ RMAppManager appManager = new RMAppManager(rmContext, scheduler, null,
mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
new EventHandler<Event>() {
@@ -558,7 +558,7 @@ public class TestClientRMService {
mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
any(QueueACL.class), any(RMApp.class), any(String.class),
any())).thenReturn(true);
- return new ClientRMService(rmContext, yarnScheduler, appManager,
+ return new ClientRMService(rmContext, scheduler, appManager,
mockAclsManager, mockQueueACLsManager, null);
}
@@ -907,9 +907,9 @@ public class TestClientRMService {
@Test
public void testGetQueueInfo() throws Exception {
- YarnScheduler yarnScheduler = mock(YarnScheduler.class);
+ ResourceScheduler scheduler = mock(ResourceScheduler.class);
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
@@ -921,7 +921,7 @@ public class TestClientRMService {
any(ApplicationAccessType.class), anyString(),
any(ApplicationId.class))).thenReturn(true);
- ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
+ ClientRMService rmService = new ClientRMService(rmContext, scheduler,
null, mockAclsManager, mockQueueACLsManager, null);
GetQueueInfoRequest request = recordFactory
.newRecordInstance(GetQueueInfoRequest.class);
@@ -960,7 +960,7 @@ public class TestClientRMService {
any(ApplicationAccessType.class), anyString(),
any(ApplicationId.class))).thenReturn(false);
- ClientRMService rmService1 = new ClientRMService(rmContext, yarnScheduler,
+ ClientRMService rmService1 = new ClientRMService(rmContext, scheduler,
null, mockAclsManager1, mockQueueACLsManager1, null);
request.setQueueName("testqueue");
request.setIncludeApplications(true);
@@ -974,12 +974,12 @@ public class TestClientRMService {
@Test (timeout = 30000)
@SuppressWarnings ("rawtypes")
public void testAppSubmit() throws Exception {
- YarnScheduler yarnScheduler = mockYarnScheduler();
+ ResourceScheduler scheduler = mockResourceScheduler();
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
when(rmContext.getStateStore()).thenReturn(stateStore);
- RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
+ RMAppManager appManager = new RMAppManager(rmContext, scheduler,
null, mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
new EventHandler<Event>() {
@@ -1001,7 +1001,7 @@ public class TestClientRMService {
any()))
.thenReturn(true);
ClientRMService rmService =
- new ClientRMService(rmContext, yarnScheduler, appManager,
+ new ClientRMService(rmContext, scheduler, appManager,
mockAclsManager, mockQueueACLsManager, null);
rmService.init(new Configuration());
@@ -1085,15 +1085,15 @@ public class TestClientRMService {
* 2. Test each of the filters
*/
// Basic setup
- YarnScheduler yarnScheduler = mockYarnScheduler();
+ ResourceScheduler scheduler = mockResourceScheduler();
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
when(rmContext.getStateStore()).thenReturn(stateStore);
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
.getRMTimelineCollectorManager();
- RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
+ RMAppManager appManager = new RMAppManager(rmContext, scheduler,
null, mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
new EventHandler<Event>() {
@@ -1107,7 +1107,7 @@ public class TestClientRMService {
any()))
.thenReturn(true);
ClientRMService rmService =
- new ClientRMService(rmContext, yarnScheduler, appManager,
+ new ClientRMService(rmContext, scheduler, appManager,
mockAclsManager, mockQueueACLsManager, null);
rmService.init(new Configuration());
@@ -1238,12 +1238,12 @@ public class TestClientRMService {
public void testConcurrentAppSubmit()
throws IOException, InterruptedException, BrokenBarrierException,
YarnException {
- YarnScheduler yarnScheduler = mockYarnScheduler();
+ ResourceScheduler scheduler = mockResourceScheduler();
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
when(rmContext.getStateStore()).thenReturn(stateStore);
- RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
+ RMAppManager appManager = new RMAppManager(rmContext, scheduler,
null, mock(ApplicationACLsManager.class), new Configuration());
final ApplicationId appId1 = getApplicationId(100);
@@ -1280,7 +1280,7 @@ public class TestClientRMService {
.getRMTimelineCollectorManager();
final ClientRMService rmService =
- new ClientRMService(rmContext, yarnScheduler, appManager, null, null,
+ new ClientRMService(rmContext, scheduler, appManager, null, null,
null);
rmService.init(new Configuration());
@@ -1339,7 +1339,7 @@ public class TestClientRMService {
return submitRequest;
}
- private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext)
+ private void mockRMContext(ResourceScheduler scheduler, RMContext rmContext)
throws IOException {
Dispatcher dispatcher = mock(Dispatcher.class);
when(rmContext.getDispatcher()).thenReturn(dispatcher);
@@ -1361,22 +1361,21 @@ public class TestClientRMService {
queueConfigsByPartition.put("*", queueConfigs);
queInfo.setQueueConfigurations(queueConfigsByPartition);
- when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
+ when(scheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
.thenReturn(queInfo);
- when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
- .thenThrow(new IOException("queue does not exist"));
+ when(scheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(),
+ anyBoolean())).thenThrow(new IOException("queue does not exist"));
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
- ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
- yarnScheduler);
+ ConcurrentHashMap<ApplicationId, RMApp> apps =
+ getRMApps(rmContext, scheduler);
when(rmContext.getRMApps()).thenReturn(apps);
- when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
+ when(scheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
getSchedulerApps(apps));
- ResourceScheduler rs = mock(ResourceScheduler.class);
- when(rmContext.getScheduler()).thenReturn(rs);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
}
private ConcurrentHashMap<ApplicationId, RMApp> getRMApps(
@@ -1480,31 +1479,32 @@ public class TestClientRMService {
return app;
}
- private static YarnScheduler mockYarnScheduler() throws YarnException {
- YarnScheduler yarnScheduler = mock(YarnScheduler.class);
- when(yarnScheduler.getMinimumResourceCapability()).thenReturn(
+ private static ResourceScheduler mockResourceScheduler()
+ throws YarnException {
+ ResourceScheduler scheduler = mock(ResourceScheduler.class);
+ when(scheduler.getMinimumResourceCapability()).thenReturn(
Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
- when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
+ when(scheduler.getMaximumResourceCapability()).thenReturn(
Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
- when(yarnScheduler.getMaximumResourceCapability(any(String.class)))
- .thenReturn(Resources.createResource(
+ when(scheduler.getMaximumResourceCapability(anyString())).thenReturn(
+ Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
- when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(
+ when(scheduler.getAppsInQueue(QUEUE_1)).thenReturn(
Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
- when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
+ when(scheduler.getAppsInQueue(QUEUE_2)).thenReturn(
Arrays.asList(getApplicationAttemptId(103)));
ApplicationAttemptId attemptId = getApplicationAttemptId(1);
- when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
+ when(scheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
ResourceCalculator rs = mock(ResourceCalculator.class);
- when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
+ when(scheduler.getResourceCalculator()).thenReturn(rs);
- when(yarnScheduler.checkAndGetApplicationPriority(any(Priority.class),
+ when(scheduler.checkAndGetApplicationPriority(any(Priority.class),
any(UserGroupInformation.class), anyString(), any(ApplicationId.class)))
.thenReturn(Priority.newInstance(0));
- return yarnScheduler;
+ return scheduler;
}
private ResourceManager setupResourceManager() {
@@ -2427,15 +2427,15 @@ public class TestClientRMService {
* Submit 3 applications alternately in two queues
*/
// Basic setup
- YarnScheduler yarnScheduler = mockYarnScheduler();
+ ResourceScheduler scheduler = mockResourceScheduler();
RMContext rmContext = mock(RMContext.class);
- mockRMContext(yarnScheduler, rmContext);
+ mockRMContext(scheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
when(rmContext.getStateStore()).thenReturn(stateStore);
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
.getRMTimelineCollectorManager();
- RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
+ RMAppManager appManager = new RMAppManager(rmContext, scheduler, null,
mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler())
.thenReturn(new EventHandler<Event>() {
@@ -2454,7 +2454,7 @@ public class TestClientRMService {
when(appAclsManager.checkAccess(eq(UserGroupInformation.getCurrentUser()),
any(ApplicationAccessType.class), any(String.class),
any(ApplicationId.class))).thenReturn(false);
- ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
+ ClientRMService rmService = new ClientRMService(rmContext, scheduler,
appManager, appAclsManager, queueAclsManager, null);
rmService.init(new Configuration());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDDS-561. Move Node2ContainerMap
and Node2PipelineMap to NodeManager. Contributed by Lokesh Jain.
Posted by in...@apache.org.
HDDS-561. Move Node2ContainerMap and Node2PipelineMap to NodeManager. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3929626
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3929626
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3929626
Branch: refs/heads/HDFS-13532
Commit: a39296260f8c77f3808e27b43c623a0edebe4a17
Parents: 81072d5
Author: Nanda kumar <na...@apache.org>
Authored: Tue Oct 2 19:47:15 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Tue Oct 2 19:47:44 2018 +0530
----------------------------------------------------------------------
.../scm/container/ContainerReportHandler.java | 16 ++--
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 11 +--
.../hadoop/hdds/scm/node/NewNodeHandler.java | 9 +-
.../hadoop/hdds/scm/node/NodeManager.java | 61 ++++++++++++++
.../hadoop/hdds/scm/node/NodeStateManager.java | 88 +++++++++++++++++++-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 83 ++++++++++++++++++
.../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 9 +-
.../hdds/scm/node/states/Node2ContainerMap.java | 2 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 75 +++++++++++++++++
.../hdds/scm/pipelines/Node2PipelineMap.java | 75 -----------------
.../hdds/scm/pipelines/PipelineSelector.java | 18 ++--
.../scm/server/StorageContainerManager.java | 14 ++--
.../hdds/scm/container/MockNodeManager.java | 87 +++++++++++++++++++
.../container/TestContainerReportHandler.java | 14 ++--
.../hdds/scm/node/TestDeadNodeHandler.java | 27 ++----
.../testutils/ReplicationNodeManagerMock.java | 82 ++++++++++++++++++
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +-
19 files changed, 520 insertions(+), 157 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index 3f156de..71935f0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hdds.scm.container.replication.ReplicationActivityStatu
import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -48,7 +48,7 @@ public class ContainerReportHandler implements
private static final Logger LOG =
LoggerFactory.getLogger(ContainerReportHandler.class);
- private final Node2ContainerMap node2ContainerMap;
+ private final NodeManager nodeManager;
private final Mapping containerMapping;
@@ -57,14 +57,14 @@ public class ContainerReportHandler implements
private ReplicationActivityStatus replicationStatus;
public ContainerReportHandler(Mapping containerMapping,
- Node2ContainerMap node2ContainerMap,
+ NodeManager nodeManager,
ReplicationActivityStatus replicationActivityStatus) {
Preconditions.checkNotNull(containerMapping);
- Preconditions.checkNotNull(node2ContainerMap);
+ Preconditions.checkNotNull(nodeManager);
Preconditions.checkNotNull(replicationActivityStatus);
this.containerStateManager = containerMapping.getStateManager();
+ this.nodeManager = nodeManager;
this.containerMapping = containerMapping;
- this.node2ContainerMap = node2ContainerMap;
this.replicationStatus = replicationActivityStatus;
}
@@ -89,11 +89,11 @@ public class ContainerReportHandler implements
.map(ContainerID::new)
.collect(Collectors.toSet());
- ReportResult<ContainerID> reportResult = node2ContainerMap
- .processReport(datanodeOrigin.getUuid(), containerIds);
+ ReportResult<ContainerID> reportResult = nodeManager
+ .processContainerReport(datanodeOrigin.getUuid(), containerIds);
//we have the report, so we can update the states for the next iteration.
- node2ContainerMap
+ nodeManager
.setContainersForDatanode(datanodeOrigin.getUuid(), containerIds);
for (ContainerID containerID : reportResult.getMissingEntries()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
index 253b3ec..17edf9e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -38,8 +37,6 @@ import org.slf4j.LoggerFactory;
*/
public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
- private final Node2ContainerMap node2ContainerMap;
-
private final ContainerStateManager containerStateManager;
private final NodeManager nodeManager;
@@ -47,10 +44,8 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
private static final Logger LOG =
LoggerFactory.getLogger(DeadNodeHandler.class);
- public DeadNodeHandler(
- Node2ContainerMap node2ContainerMap,
- ContainerStateManager containerStateManager, NodeManager nodeManager) {
- this.node2ContainerMap = node2ContainerMap;
+ public DeadNodeHandler(NodeManager nodeManager,
+ ContainerStateManager containerStateManager) {
this.containerStateManager = containerStateManager;
this.nodeManager = nodeManager;
}
@@ -61,7 +56,7 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
nodeManager.processDeadNode(datanodeDetails.getUuid());
Set<ContainerID> containers =
- node2ContainerMap.getContainers(datanodeDetails.getUuid());
+ nodeManager.getContainers(datanodeDetails.getUuid());
if (containers == null) {
LOG.info("There's no containers in dead datanode {}, no replica will be"
+ " removed from the in-memory state.", datanodeDetails.getUuid());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
index 79b75a5..780aa2b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NewNodeHandler.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -31,17 +30,17 @@ import java.util.Collections;
*/
public class NewNodeHandler implements EventHandler<DatanodeDetails> {
- private final Node2ContainerMap node2ContainerMap;
+ private final NodeManager nodeManager;
- public NewNodeHandler(Node2ContainerMap node2ContainerMap) {
- this.node2ContainerMap = node2ContainerMap;
+ public NewNodeHandler(NodeManager nodeManager) {
+ this.nodeManager = nodeManager;
}
@Override
public void onMessage(DatanodeDetails datanodeDetails,
EventPublisher publisher) {
try {
- node2ContainerMap.insertNewDatanode(datanodeDetails.getUuid(),
+ nodeManager.addDatanodeInContainerMap(datanodeDetails.getUuid(),
Collections.emptySet());
} catch (SCMException e) {
// TODO: log exception message.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index fad3ee3..0dc1a0c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -18,11 +18,16 @@
package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -31,6 +36,7 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import java.io.Closeable;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
/**
@@ -134,6 +140,61 @@ public interface NodeManager extends StorageContainerNodeProtocol,
NodeState getNodeState(DatanodeDetails datanodeDetails);
/**
+ * Get set of pipelines a datanode is part of.
+ * @param dnId - datanodeID
+ * @return Set of PipelineID
+ */
+ Set<PipelineID> getPipelineByDnID(UUID dnId);
+
+ /**
+ * Add pipeline information in the NodeManager.
+ * @param pipeline - Pipeline to be added
+ */
+ void addPipeline(Pipeline pipeline);
+
+ /**
+ * Remove a pipeline information from the NodeManager.
+ * @param pipeline - Pipeline to be removed
+ */
+ void removePipeline(Pipeline pipeline);
+
+ /**
+ * Update set of containers available on a datanode.
+ * @param uuid - DatanodeID
+ * @param containerIds - Set of containerIDs
+ * @throws SCMException - if datanode is not known. For new datanode use
+ * addDatanodeInContainerMap call.
+ */
+ void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
+ throws SCMException;
+
+ /**
+ * Process containerReport received from datanode.
+ * @param uuid - DataonodeID
+ * @param containerIds - Set of containerIDs
+ * @return The result after processing containerReport
+ */
+ ReportResult<ContainerID> processContainerReport(UUID uuid,
+ Set<ContainerID> containerIds);
+
+ /**
+ * Return set of containerIDs available on a datanode.
+ * @param uuid - DatanodeID
+ * @return - set of containerIDs
+ */
+ Set<ContainerID> getContainers(UUID uuid);
+
+ /**
+ * Insert a new datanode with set of containerIDs for containers available
+ * on it.
+ * @param uuid - DatanodeID
+ * @param containerIDs - Set of ContainerIDs
+ * @throws SCMException - if datanode already exists
+ */
+ void addDatanodeInContainerMap(UUID uuid, Set<ContainerID> containerIDs)
+ throws SCMException;
+
+ /**
* Add a {@link SCMCommand} to the command queue, which are
* handled by HB thread asynchronously.
* @param dnId datanode uuid
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index 1f99ffe..88f984b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@ -24,11 +24,14 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.HddsServerUtil;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
-import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
-import org.apache.hadoop.hdds.scm.node.states.NodeStateMap;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.states.*;
+import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
import org.apache.hadoop.hdds.server.events.Event;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.common.statemachine
@@ -87,6 +90,15 @@ public class NodeStateManager implements Runnable, Closeable {
*/
private final NodeStateMap nodeStateMap;
/**
+ * Maintains the mapping from node to pipelines a node is part of.
+ */
+ private final Node2PipelineMap node2PipelineMap;
+ /**
+ * Maintains the map from node to ContainerIDs for the containers
+ * available on the node.
+ */
+ private final Node2ContainerMap node2ContainerMap;
+ /**
* Used for publishing node state change events.
*/
private final EventPublisher eventPublisher;
@@ -118,6 +130,8 @@ public class NodeStateManager implements Runnable, Closeable {
*/
public NodeStateManager(Configuration conf, EventPublisher eventPublisher) {
this.nodeStateMap = new NodeStateMap();
+ this.node2PipelineMap = new Node2PipelineMap();
+ this.node2ContainerMap = new Node2ContainerMap();
this.eventPublisher = eventPublisher;
this.state2EventMap = new HashMap<>();
initialiseState2EventMap();
@@ -243,6 +257,14 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
+ * Adds a pipeline in the node2PipelineMap.
+ * @param pipeline - Pipeline to be added
+ */
+ public void addPipeline(Pipeline pipeline) {
+ node2PipelineMap.addPipeline(pipeline);
+ }
+
+ /**
* Get information about the node.
*
* @param datanodeDetails DatanodeDetails
@@ -353,6 +375,15 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
+ * Gets set of pipelineID a datanode belongs to.
+ * @param dnId - Datanode ID
+ * @return Set of PipelineID
+ */
+ public Set<PipelineID> getPipelineByDnID(UUID dnId) {
+ return node2PipelineMap.getPipelines(dnId);
+ }
+
+ /**
* Returns the count of healthy nodes.
*
* @return healthy node count
@@ -457,6 +488,57 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
+ * Removes a pipeline from the node2PipelineMap.
+ * @param pipeline - Pipeline to be removed
+ */
+ public void removePipeline(Pipeline pipeline) {
+ node2PipelineMap.removePipeline(pipeline);
+ }
+ /**
+ * Update set of containers available on a datanode.
+ * @param uuid - DatanodeID
+ * @param containerIds - Set of containerIDs
+ * @throws SCMException - if datanode is not known. For new datanode use
+ * addDatanodeInContainerMap call.
+ */
+ public void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
+ throws SCMException {
+ node2ContainerMap.setContainersForDatanode(uuid, containerIds);
+ }
+
+ /**
+ * Process containerReport received from datanode.
+ * @param uuid - DataonodeID
+ * @param containerIds - Set of containerIDs
+ * @return The result after processing containerReport
+ */
+ public ReportResult<ContainerID> processContainerReport(UUID uuid,
+ Set<ContainerID> containerIds) {
+ return node2ContainerMap.processReport(uuid, containerIds);
+ }
+
+ /**
+ * Return set of containerIDs available on a datanode.
+ * @param uuid - DatanodeID
+ * @return - set of containerIDs
+ */
+ public Set<ContainerID> getContainers(UUID uuid) {
+ return node2ContainerMap.getContainers(uuid);
+ }
+
+ /**
+ * Insert a new datanode with set of containerIDs for containers available
+ * on it.
+ * @param uuid - DatanodeID
+ * @param containerIDs - Set of ContainerIDs
+ * @throws SCMException - if datanode already exists
+ */
+ public void addDatanodeInContainerMap(UUID uuid,
+ Set<ContainerID> containerIDs) throws SCMException {
+ node2ContainerMap.insertNewDatanode(uuid, containerIDs);
+ }
+
+ /**
* Move Stale or Dead node to healthy if we got a heartbeat from them.
* Move healthy nodes to stale nodes if it is needed.
* Move Stales node to dead if needed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 620c816..36a6f15 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -21,8 +21,13 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.VersionInfo;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -59,6 +64,7 @@ import java.net.InetAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -471,6 +477,83 @@ public class SCMNodeManager
return nodeCountMap;
}
+ /**
+ * Get set of pipelines a datanode is part of.
+ * @param dnId - datanodeID
+ * @return Set of PipelineID
+ */
+ @Override
+ public Set<PipelineID> getPipelineByDnID(UUID dnId) {
+ return nodeStateManager.getPipelineByDnID(dnId);
+ }
+
+
+ /**
+ * Add pipeline information in the NodeManager.
+ * @param pipeline - Pipeline to be added
+ */
+ @Override
+ public void addPipeline(Pipeline pipeline) {
+ nodeStateManager.addPipeline(pipeline);
+ }
+
+ /**
+ * Remove a pipeline information from the NodeManager.
+ * @param pipeline - Pipeline to be removed
+ */
+ @Override
+ public void removePipeline(Pipeline pipeline) {
+ nodeStateManager.removePipeline(pipeline);
+ }
+
+ /**
+ * Update set of containers available on a datanode.
+ * @param uuid - DatanodeID
+ * @param containerIds - Set of containerIDs
+ * @throws SCMException - if datanode is not known. For new datanode use
+ * addDatanodeInContainerMap call.
+ */
+ @Override
+ public void setContainersForDatanode(UUID uuid,
+ Set<ContainerID> containerIds) throws SCMException {
+ nodeStateManager.setContainersForDatanode(uuid, containerIds);
+ }
+
+ /**
+ * Process containerReport received from datanode.
+ * @param uuid - DataonodeID
+ * @param containerIds - Set of containerIDs
+ * @return The result after processing containerReport
+ */
+ @Override
+ public ReportResult<ContainerID> processContainerReport(UUID uuid,
+ Set<ContainerID> containerIds) {
+ return nodeStateManager.processContainerReport(uuid, containerIds);
+ }
+
+ /**
+ * Return set of containerIDs available on a datanode.
+ * @param uuid - DatanodeID
+ * @return - set of containerIDs
+ */
+ @Override
+ public Set<ContainerID> getContainers(UUID uuid) {
+ return nodeStateManager.getContainers(uuid);
+ }
+
+ /**
+ * Insert a new datanode with set of containerIDs for containers available
+ * on it.
+ * @param uuid - DatanodeID
+ * @param containerIDs - Set of ContainerIDs
+ * @throws SCMException - if datanode already exists
+ */
+ @Override
+ public void addDatanodeInContainerMap(UUID uuid,
+ Set<ContainerID> containerIDs) throws SCMException {
+ nodeStateManager.addDatanodeInContainerMap(uuid, containerIDs);
+ }
+
// TODO:
// Since datanode commands are added through event queue, onMessage method
// should take care of adding commands to command queue.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
index 6ea83df..1b0e5b5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
@@ -147,7 +147,7 @@ public class SCMNodeStorageStatMap implements SCMNodeStorageStatMXBean {
* @param datanodeID - UUID of DN.
* @param report - set of Storage Reports for the Datanode.
* @throws SCMException - if we don't know about this datanode, for new DN
- * use insertNewDatanode.
+ * use addDatanodeInContainerMap.
*/
public void updateDatanodeMap(UUID datanodeID,
Set<StorageLocationReport> report) throws SCMException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
index ddbba82..48939f1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
@@ -19,25 +19,18 @@
package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Handles Stale node event.
*/
public class StaleNodeHandler implements EventHandler<DatanodeDetails> {
- static final Logger LOG = LoggerFactory.getLogger(StaleNodeHandler.class);
- private final Node2ContainerMap node2ContainerMap;
private final PipelineSelector pipelineSelector;
- public StaleNodeHandler(Node2ContainerMap node2ContainerMap,
- PipelineSelector pipelineSelector) {
- this.node2ContainerMap = node2ContainerMap;
+ public StaleNodeHandler(PipelineSelector pipelineSelector) {
this.pipelineSelector = pipelineSelector;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
index 549080a..9625f81 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
@@ -70,7 +70,7 @@ public class Node2ContainerMap extends Node2ObjectsMap<ContainerID> {
* @param datanodeID - UUID of DN.
* @param containers - Set of Containers tht is present on DN.
* @throws SCMException - if we don't know about this datanode, for new DN
- * use insertNewDatanode.
+ * use addDatanodeInContainerMap.
*/
public void setContainersForDatanode(UUID datanodeID,
Set<ContainerID> containers) throws SCMException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
new file mode 100644
index 0000000..87f2222
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.scm.node.states;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * This data structure maintains the list of pipelines which the given datanode is a part of. This
+ * information will be added whenever a new pipeline allocation happens.
+ *
+ * <p>TODO: this information needs to be regenerated from pipeline reports on SCM restart
+ */
+public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
+
+ /** Constructs a Node2PipelineMap Object. */
+ public Node2PipelineMap() {
+ super();
+ }
+
+ /**
+ * Returns null if there no pipelines associated with this datanode ID.
+ *
+ * @param datanode - UUID
+ * @return Set of pipelines or Null.
+ */
+ public Set<PipelineID> getPipelines(UUID datanode) {
+ return getObjects(datanode);
+ }
+
+ /**
+ * Adds a pipeline entry to a given dataNode in the map.
+ *
+ * @param pipeline Pipeline to be added
+ */
+ public synchronized void addPipeline(Pipeline pipeline) {
+ for (DatanodeDetails details : pipeline.getDatanodes().values()) {
+ UUID dnId = details.getUuid();
+ dn2ObjectMap.computeIfAbsent(dnId, k -> new HashSet<>())
+ .add(pipeline.getId());
+ }
+ }
+
+ public synchronized void removePipeline(Pipeline pipeline) {
+ for (DatanodeDetails details : pipeline.getDatanodes().values()) {
+ UUID dnId = details.getUuid();
+ dn2ObjectMap.computeIfPresent(dnId,
+ (k, v) -> {
+ v.remove(pipeline.getId());
+ return v;
+ });
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
deleted file mode 100644
index 87f2222..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.node.states;
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.UUID;
-
-/**
- * This data structure maintains the list of pipelines which the given datanode is a part of. This
- * information will be added whenever a new pipeline allocation happens.
- *
- * <p>TODO: this information needs to be regenerated from pipeline reports on SCM restart
- */
-public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
-
- /** Constructs a Node2PipelineMap Object. */
- public Node2PipelineMap() {
- super();
- }
-
- /**
- * Returns null if there no pipelines associated with this datanode ID.
- *
- * @param datanode - UUID
- * @return Set of pipelines or Null.
- */
- public Set<PipelineID> getPipelines(UUID datanode) {
- return getObjects(datanode);
- }
-
- /**
- * Adds a pipeline entry to a given dataNode in the map.
- *
- * @param pipeline Pipeline to be added
- */
- public synchronized void addPipeline(Pipeline pipeline) {
- for (DatanodeDetails details : pipeline.getDatanodes().values()) {
- UUID dnId = details.getUuid();
- dn2ObjectMap.computeIfAbsent(dnId, k -> new HashSet<>())
- .add(pipeline.getId());
- }
- }
-
- public synchronized void removePipeline(Pipeline pipeline) {
- for (DatanodeDetails details : pipeline.getDatanodes().values()) {
- UUID dnId = details.getUuid();
- dn2ObjectMap.computeIfPresent(dnId,
- (k, v) -> {
- v.remove(pipeline.getId());
- return v;
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
index 59d937e..c8d22ff 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
import org.apache.hadoop.hdds.scm.pipelines.ratis.RatisManagerImpl;
import org.apache.hadoop.hdds.scm.pipelines.standalone.StandaloneManagerImpl;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -61,7 +60,6 @@ import java.util.HashMap;
import java.util.Set;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
@@ -85,7 +83,7 @@ public class PipelineSelector {
private final long containerSize;
private final MetadataStore pipelineStore;
private final PipelineStateManager stateManager;
- private final Node2PipelineMap node2PipelineMap;
+ private final NodeManager nodeManager;
private final Map<PipelineID, HashSet<ContainerID>> pipeline2ContainerMap;
private final Map<PipelineID, Pipeline> pipelineMap;
private final LeaseManager<Pipeline> pipelineLeaseManager;
@@ -105,7 +103,6 @@ public class PipelineSelector {
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
StorageUnit.BYTES);
- node2PipelineMap = new Node2PipelineMap();
pipelineMap = new ConcurrentHashMap<>();
pipelineManagerMap = new HashMap<>();
@@ -124,6 +121,7 @@ public class PipelineSelector {
pipelineLeaseManager.start();
stateManager = new PipelineStateManager();
+ this.nodeManager = nodeManager;
pipeline2ContainerMap = new HashMap<>();
// Write the container name to pipeline mapping.
@@ -361,10 +359,6 @@ public class PipelineSelector {
}
}
- public Set<PipelineID> getPipelineByDnID(UUID dnId) {
- return node2PipelineMap.getPipelines(dnId);
- }
-
private void addExistingPipeline(Pipeline pipeline) throws IOException {
LifeCycleState state = pipeline.getLifeCycleState();
switch (state) {
@@ -379,7 +373,7 @@ public class PipelineSelector {
// when all the nodes have reported.
pipelineMap.put(pipeline.getId(), pipeline);
pipeline2ContainerMap.put(pipeline.getId(), new HashSet<>());
- node2PipelineMap.addPipeline(pipeline);
+ nodeManager.addPipeline(pipeline);
// reset the datanodes in the pipeline
// they will be reset on
pipeline.resetPipeline();
@@ -393,7 +387,7 @@ public class PipelineSelector {
}
public void handleStaleNode(DatanodeDetails dn) {
- Set<PipelineID> pipelineIDs = getPipelineByDnID(dn.getUuid());
+ Set<PipelineID> pipelineIDs = nodeManager.getPipelineByDnID(dn.getUuid());
for (PipelineID id : pipelineIDs) {
LOG.info("closing pipeline {}.", id);
eventPublisher.fireEvent(SCMEvents.PIPELINE_CLOSE, id);
@@ -436,7 +430,7 @@ public class PipelineSelector {
case CREATE:
pipelineMap.put(pipeline.getId(), pipeline);
pipeline2ContainerMap.put(pipeline.getId(), new HashSet<>());
- node2PipelineMap.addPipeline(pipeline);
+ nodeManager.addPipeline(pipeline);
// Acquire lease on pipeline
Lease<Pipeline> pipelineLease = pipelineLeaseManager.acquire(pipeline);
// Register callback to be executed in case of timeout
@@ -459,7 +453,7 @@ public class PipelineSelector {
case TIMEOUT:
closePipeline(pipeline);
pipeline2ContainerMap.remove(pipeline.getId());
- node2PipelineMap.removePipeline(pipeline);
+ nodeManager.removePipeline(pipeline);
pipelineMap.remove(pipeline.getId());
break;
default:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index bb72075..a6a967c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -62,7 +62,6 @@ import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeReportHandler;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.scm.node.StaleNodeHandler;
-import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineCloseHandler;
import org.apache.hadoop.hdds.scm.pipelines.PipelineActionEventHandler;
import org.apache.hadoop.hdds.scm.pipelines.PipelineReportHandler;
@@ -212,8 +211,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmBlockManager = new BlockManagerImpl(
conf, getScmNodeManager(), scmContainerManager, eventQueue);
- Node2ContainerMap node2ContainerMap = new Node2ContainerMap();
-
replicationStatus = new ReplicationActivityStatus();
CloseContainerEventHandler closeContainerHandler =
@@ -226,18 +223,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
CommandStatusReportHandler cmdStatusReportHandler =
new CommandStatusReportHandler();
- NewNodeHandler newNodeHandler = new NewNodeHandler(node2ContainerMap);
+ NewNodeHandler newNodeHandler = new NewNodeHandler(scmNodeManager);
StaleNodeHandler staleNodeHandler =
- new StaleNodeHandler(node2ContainerMap,
- scmContainerManager.getPipelineSelector());
- DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap,
- getScmContainerManager().getStateManager(), scmNodeManager);
+ new StaleNodeHandler(scmContainerManager.getPipelineSelector());
+ DeadNodeHandler deadNodeHandler = new DeadNodeHandler(scmNodeManager,
+ getScmContainerManager().getStateManager());
ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
PendingDeleteHandler pendingDeleteHandler =
new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService());
ContainerReportHandler containerReportHandler =
- new ContainerReportHandler(scmContainerManager, node2ContainerMap,
+ new ContainerReportHandler(scmContainerManager, scmNodeManager,
replicationStatus);
scmChillModeManager = new SCMChillModeManager(conf,
getScmContainerManager().getStateManager().getAllContainers(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 0d90728..3221053 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -19,9 +19,14 @@ package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
+import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -29,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
+import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.VersionResponse;
@@ -42,6 +48,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
@@ -73,12 +80,16 @@ public class MockNodeManager implements NodeManager {
private final SCMNodeStat aggregateStat;
private boolean chillmode;
private final Map<UUID, List<SCMCommand>> commandMap;
+ private final Node2PipelineMap node2PipelineMap;
+ private final Node2ContainerMap node2ContainerMap;
public MockNodeManager(boolean initializeFakeNodes, int nodeCount) {
this.healthyNodes = new LinkedList<>();
this.staleNodes = new LinkedList<>();
this.deadNodes = new LinkedList<>();
this.nodeMetricMap = new HashMap<>();
+ this.node2PipelineMap = new Node2PipelineMap();
+ this.node2ContainerMap = new Node2ContainerMap();
aggregateStat = new SCMNodeStat();
if (initializeFakeNodes) {
for (int x = 0; x < nodeCount; x++) {
@@ -289,6 +300,34 @@ public class MockNodeManager implements NodeManager {
return null;
}
+ /**
+ * Get set of pipelines a datanode is part of.
+ * @param dnId - datanodeID
+ * @return Set of PipelineID
+ */
+ @Override
+ public Set<PipelineID> getPipelineByDnID(UUID dnId) {
+ return node2PipelineMap.getPipelines(dnId);
+ }
+
+ /**
+ * Add pipeline information in the NodeManager.
+ * @param pipeline - Pipeline to be added
+ */
+ @Override
+ public void addPipeline(Pipeline pipeline) {
+ node2PipelineMap.addPipeline(pipeline);
+ }
+
+ /**
+ * Remove a pipeline information from the NodeManager.
+ * @param pipeline - Pipeline to be removed
+ */
+ @Override
+ public void removePipeline(Pipeline pipeline) {
+ node2PipelineMap.removePipeline(pipeline);
+ }
+
@Override
public void addDatanodeCommand(UUID dnId, SCMCommand command) {
if(commandMap.containsKey(dnId)) {
@@ -313,6 +352,54 @@ public class MockNodeManager implements NodeManager {
// do nothing
}
+ /**
+ * Update set of containers available on a datanode.
+ * @param uuid - DatanodeID
+ * @param containerIds - Set of containerIDs
+ * @throws SCMException - if datanode is not known. For new datanode use
+ * addDatanodeInContainerMap call.
+ */
+ @Override
+ public void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
+ throws SCMException {
+ node2ContainerMap.setContainersForDatanode(uuid, containerIds);
+ }
+
+ /**
+ * Process containerReport received from datanode.
+ * @param uuid - DataonodeID
+ * @param containerIds - Set of containerIDs
+ * @return The result after processing containerReport
+ */
+ @Override
+ public ReportResult<ContainerID> processContainerReport(UUID uuid,
+ Set<ContainerID> containerIds) {
+ return node2ContainerMap.processReport(uuid, containerIds);
+ }
+
+ /**
+ * Return set of containerIDs available on a datanode.
+ * @param uuid - DatanodeID
+ * @return - set of containerIDs
+ */
+ @Override
+ public Set<ContainerID> getContainers(UUID uuid) {
+ return node2ContainerMap.getContainers(uuid);
+ }
+
+ /**
+ * Insert a new datanode with set of containerIDs for containers available
+ * on it.
+ * @param uuid - DatanodeID
+ * @param containerIDs - Set of ContainerIDs
+ * @throws SCMException - if datanode already exists
+ */
+ @Override
+ public void addDatanodeInContainerMap(UUID uuid,
+ Set<ContainerID> containerIDs) throws SCMException {
+ node2ContainerMap.insertNewDatanode(uuid, containerIDs);
+ }
+
// Returns the number of commands that is queued to this node manager.
public int getCommandCount(DatanodeDetails dd) {
List<SCMCommand> list = commandMap.get(dd.getUuid());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index a59179b..f79ae1e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.container.replication
.ReplicationActivityStatus;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
-import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.ContainerReportFromDatanode;
@@ -60,6 +60,7 @@ import org.slf4j.LoggerFactory;
public class TestContainerReportHandler implements EventPublisher {
private List<Object> publishedEvents = new ArrayList<>();
+ private final NodeManager nodeManager = new MockNodeManager(true, 1);
private static final Logger LOG =
LoggerFactory.getLogger(TestContainerReportHandler.class);
@@ -73,7 +74,6 @@ public class TestContainerReportHandler implements EventPublisher {
public void test() throws IOException {
//GIVEN
OzoneConfiguration conf = new OzoneConfiguration();
- Node2ContainerMap node2ContainerMap = new Node2ContainerMap();
Mapping mapping = Mockito.mock(Mapping.class);
PipelineSelector selector = Mockito.mock(PipelineSelector.class);
@@ -96,17 +96,17 @@ public class TestContainerReportHandler implements EventPublisher {
new ReplicationActivityStatus();
ContainerReportHandler reportHandler =
- new ContainerReportHandler(mapping, node2ContainerMap,
+ new ContainerReportHandler(mapping, nodeManager,
replicationActivityStatus);
DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
DatanodeDetails dn2 = TestUtils.randomDatanodeDetails();
DatanodeDetails dn3 = TestUtils.randomDatanodeDetails();
DatanodeDetails dn4 = TestUtils.randomDatanodeDetails();
- node2ContainerMap.insertNewDatanode(dn1.getUuid(), new HashSet<>());
- node2ContainerMap.insertNewDatanode(dn2.getUuid(), new HashSet<>());
- node2ContainerMap.insertNewDatanode(dn3.getUuid(), new HashSet<>());
- node2ContainerMap.insertNewDatanode(dn4.getUuid(), new HashSet<>());
+ nodeManager.addDatanodeInContainerMap(dn1.getUuid(), new HashSet<>());
+ nodeManager.addDatanodeInContainerMap(dn2.getUuid(), new HashSet<>());
+ nodeManager.addDatanodeInContainerMap(dn3.getUuid(), new HashSet<>());
+ nodeManager.addDatanodeInContainerMap(dn4.getUuid(), new HashSet<>());
PipelineSelector pipelineSelector = Mockito.mock(PipelineSelector.class);
Pipeline pipeline = new Pipeline("leader", LifeCycleState.CLOSED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 6966322..7bba032 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
@@ -60,7 +59,6 @@ public class TestDeadNodeHandler {
private List<ReplicationRequest> sentEvents = new ArrayList<>();
private SCMNodeManager nodeManager;
- private Node2ContainerMap node2ContainerMap;
private ContainerStateManager containerStateManager;
private NodeReportHandler nodeReportHandler;
private DeadNodeHandler deadNodeHandler;
@@ -70,14 +68,13 @@ public class TestDeadNodeHandler {
@Before
public void setup() throws IOException {
OzoneConfiguration conf = new OzoneConfiguration();
- node2ContainerMap = new Node2ContainerMap();
containerStateManager = new ContainerStateManager(conf,
Mockito.mock(Mapping.class),
Mockito.mock(PipelineSelector.class));
eventQueue = new EventQueue();
nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue);
- deadNodeHandler = new DeadNodeHandler(node2ContainerMap,
- containerStateManager, nodeManager);
+ deadNodeHandler = new DeadNodeHandler(nodeManager,
+ containerStateManager);
eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
publisher = Mockito.mock(EventPublisher.class);
nodeReportHandler = new NodeReportHandler(nodeManager);
@@ -96,8 +93,8 @@ public class TestDeadNodeHandler {
ContainerInfo container3 =
TestUtils.allocateContainer(containerStateManager);
- registerReplicas(node2ContainerMap, datanode1, container1, container2);
- registerReplicas(node2ContainerMap, datanode2, container1, container3);
+ registerReplicas(datanode1, container1, container2);
+ registerReplicas(datanode2, container1, container3);
registerReplicas(containerStateManager, container1, datanode1, datanode2);
registerReplicas(containerStateManager, container2, datanode1);
@@ -105,13 +102,8 @@ public class TestDeadNodeHandler {
TestUtils.closeContainer(containerStateManager, container1);
- //WHEN datanode1 is dead
deadNodeHandler.onMessage(datanode1, publisher);
- //THEN
- //node2ContainerMap has not been changed
- Assert.assertEquals(2, node2ContainerMap.size());
-
Set<DatanodeDetails> container1Replicas =
containerStateManager.getContainerStateMap()
.getContainerReplicas(new ContainerID(container1.getContainerID()));
@@ -168,7 +160,7 @@ public class TestDeadNodeHandler {
ContainerInfo container1 =
TestUtils.allocateContainer(containerStateManager);
- registerReplicas(node2ContainerMap, datanode1, container1);
+ registerReplicas(datanode1, container1);
SCMNodeStat stat = nodeManager.getStats();
Assert.assertTrue(stat.getCapacity().get() == 300);
@@ -211,7 +203,7 @@ public class TestDeadNodeHandler {
ContainerInfo container1 =
TestUtils.allocateContainer(containerStateManager);
- registerReplicas(node2ContainerMap, dn1, container1);
+ registerReplicas(dn1, container1);
deadNodeHandler.onMessage(dn1, eventQueue);
Assert.assertTrue(logCapturer.getOutput().contains(
@@ -226,12 +218,11 @@ public class TestDeadNodeHandler {
datanodes);
}
- private void registerReplicas(Node2ContainerMap node2ConMap,
- DatanodeDetails datanode,
+ private void registerReplicas(DatanodeDetails datanode,
ContainerInfo... containers)
throws SCMException {
- node2ConMap
- .insertNewDatanode(datanode.getUuid(),
+ nodeManager
+ .addDatanodeInContainerMap(datanode.getUuid(),
Arrays.stream(containers)
.map(container -> new ContainerID(container.getContainerID()))
.collect(Collectors.toSet()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index a9a00ef..74c3932 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -19,8 +19,12 @@ package org.apache.hadoop.ozone.container.testutils;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.CommandQueue;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
@@ -30,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
+import org.apache.hadoop.hdds.scm.node.states.ReportResult;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.ozone.protocol.VersionResponse;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -39,6 +44,7 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
/**
@@ -217,6 +223,82 @@ public class ReplicationNodeManagerMock implements NodeManager {
}
/**
+ * Get set of pipelines a datanode is part of.
+ * @param dnId - datanodeID
+ * @return Set of PipelineID
+ */
+ @Override
+ public Set<PipelineID> getPipelineByDnID(UUID dnId) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
+ * Add pipeline information in the NodeManager.
+ * @param pipeline - Pipeline to be added
+ */
+ @Override
+ public void addPipeline(Pipeline pipeline) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
+ * Remove a pipeline information from the NodeManager.
+ * @param pipeline - Pipeline to be removed
+ */
+ @Override
+ public void removePipeline(Pipeline pipeline) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
+ * Update set of containers available on a datanode.
+ * @param uuid - DatanodeID
+ * @param containerIds - Set of containerIDs
+ * @throws SCMException - if datanode is not known. For new datanode use
+ * addDatanodeInContainerMap call.
+ */
+ @Override
+ public void setContainersForDatanode(UUID uuid, Set<ContainerID> containerIds)
+ throws SCMException {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
+ * Process containerReport received from datanode.
+ * @param uuid - DataonodeID
+ * @param containerIds - Set of containerIDs
+ * @return The result after processing containerReport
+ */
+ @Override
+ public ReportResult<ContainerID> processContainerReport(UUID uuid,
+ Set<ContainerID> containerIds) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
+ * Return set of containerIDs available on a datanode.
+ * @param uuid - DatanodeID
+ * @return - set of containerIDs
+ */
+ @Override
+ public Set<ContainerID> getContainers(UUID uuid) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
+ * Insert a new datanode with set of containerIDs for containers available
+ * on it.
+ * @param uuid - DatanodeID
+ * @param containerIDs - Set of ContainerIDs
+ * @throws SCMException - if datanode already exists
+ */
+ @Override
+ public void addDatanodeInContainerMap(UUID uuid,
+ Set<ContainerID> containerIDs) throws SCMException {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
* Closes this stream and releases any system resources associated
* with it. If the stream is already closed then invoking this
* method has no effect.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
index ad3798e..e61a909 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
@@ -97,7 +97,7 @@ public class TestNode2PipelineMap {
Assert.assertEquals(3, dns.size());
// get pipeline details by dnid
- Set<PipelineID> pipelines = mapping.getPipelineSelector()
+ Set<PipelineID> pipelines = scm.getScmNodeManager()
.getPipelineByDnID(dns.get(0).getUuid());
Assert.assertEquals(1, pipelines.size());
pipelines.forEach(p -> Assert.assertEquals(p,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3929626/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
index 5eabfb9..b02eae2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
@@ -119,7 +119,7 @@ public class TestPipelineClose {
HddsProtos.LifeCycleState.CLOSED);
for (DatanodeDetails dn : ratisContainer1.getPipeline().getMachines()) {
// Assert that the pipeline has been removed from Node2PipelineMap as well
- Assert.assertEquals(pipelineSelector.getPipelineByDnID(
+ Assert.assertEquals(scm.getScmNodeManager().getPipelineByDnID(
dn.getUuid()).size(), 0);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8752. yarn-registry.md has
wrong word ong-lived, it should be long-lived. Contributed by leiqiang.
Posted by in...@apache.org.
YARN-8752. yarn-registry.md has wrong word ong-lived, it should be long-lived. Contributed by leiqiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94b5c54e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94b5c54e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94b5c54e
Branch: refs/heads/HDFS-13532
Commit: 94b5c54e8af51d178474d2f0eb5505366750ecf4
Parents: 3928af9
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Sep 26 09:42:53 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Sep 26 09:42:53 2018 +0900
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b5c54e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
index 4973862..650682f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
@@ -85,7 +85,7 @@ container ID.
## The binding problem
Hadoop YARN allows applications to run on the Hadoop cluster. Some of these are
batch jobs or queries that can managed via YARN’s existing API using its
-application ID. In addition YARN can deploy ong-lived services instances such a
+application ID. In addition YARN can deploy long-lived services instances such a
pool of Apache Tomcat web servers or an Apache HBase cluster. YARN will deploy
them across the cluster depending on the individual each component requirements
and server availability. These service instances need to be discovered by
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HADOOP-15764. Addendum patch: Fix
NPE in SecurityUtil.
Posted by in...@apache.org.
HADOOP-15764. Addendum patch: Fix NPE in SecurityUtil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c8d9076
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c8d9076
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c8d9076
Branch: refs/heads/HDFS-13532
Commit: 5c8d90763c52f6bf5224b59738739bd2d1a4b4b8
Parents: 2a5d431
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Sep 28 10:02:56 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Sep 28 10:03:22 2018 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/security/SecurityUtil.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c8d9076/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 9fea535..2313119 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -586,12 +586,14 @@ public final class SecurityUtil {
* hadoop.security.token.service.use_ip=false
*/
protected static class QualifiedHostResolver implements HostResolver {
- private List<String> searchDomains;
+ private List<String> searchDomains = new ArrayList<>();
{
ResolverConfig resolverConfig = ResolverConfig.getCurrentConfig();
- searchDomains = new ArrayList<>();
- for (Name name : resolverConfig.searchPath()) {
- searchDomains.add(name.toString());
+ Name[] names = resolverConfig.searchPath();
+ if (names != null) {
+ for (Name name : names) {
+ searchDomains.add(name.toString());
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDDS-391. Simplify Audit Framework
to make audit logging easier to use. Contributed by Dinesh Chitlangia.
Posted by in...@apache.org.
HDDS-391. Simplify Audit Framework to make audit logging easier to use.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0da03f8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0da03f8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0da03f8b
Branch: refs/heads/HDFS-13532
Commit: 0da03f8b14ed90c22137260d7e4029057421f0d8
Parents: 35c7351
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Sep 28 14:00:24 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Sep 28 14:00:24 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/ozone/audit/AuditLogger.java | 44 ++--------
.../apache/hadoop/ozone/audit/AuditMessage.java | 77 +++++++++++++++--
.../org/apache/hadoop/ozone/audit/Auditor.java | 33 +++++++
.../apache/hadoop/ozone/audit/package-info.java | 16 ++--
.../ozone/audit/TestOzoneAuditLogger.java | 89 +++++++++----------
.../common/src/test/resources/log4j2.properties | 4 +-
.../apache/hadoop/ozone/om/OzoneManager.java | 90 +++++++++++---------
7 files changed, 213 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da03f8b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
index ee20c66..9357774 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
@@ -56,53 +56,21 @@ public class AuditLogger {
}
public void logWriteSuccess(AuditMessage msg) {
- logWriteSuccess(Level.INFO, msg);
- }
-
- public void logWriteSuccess(Level level, AuditMessage msg) {
- this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, null);
+ this.logger.logIfEnabled(FQCN, Level.INFO, WRITE_MARKER, msg, null);
}
public void logWriteFailure(AuditMessage msg) {
- logWriteFailure(Level.ERROR, msg);
- }
-
- public void logWriteFailure(Level level, AuditMessage msg) {
- logWriteFailure(level, msg, null);
- }
-
- public void logWriteFailure(AuditMessage msg, Throwable exception) {
- logWriteFailure(Level.ERROR, msg, exception);
- }
-
- public void logWriteFailure(Level level, AuditMessage msg,
- Throwable exception) {
- this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, exception);
+ this.logger.logIfEnabled(FQCN, Level.ERROR, WRITE_MARKER, msg,
+ msg.getThrowable());
}
public void logReadSuccess(AuditMessage msg) {
- logReadSuccess(Level.INFO, msg);
- }
-
- public void logReadSuccess(Level level, AuditMessage msg) {
- this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, null);
+ this.logger.logIfEnabled(FQCN, Level.INFO, READ_MARKER, msg, null);
}
public void logReadFailure(AuditMessage msg) {
- logReadFailure(Level.ERROR, msg);
- }
-
- public void logReadFailure(Level level, AuditMessage msg) {
- logReadFailure(level, msg, null);
- }
-
- public void logReadFailure(AuditMessage msg, Throwable exception) {
- logReadFailure(Level.ERROR, msg, exception);
- }
-
- public void logReadFailure(Level level, AuditMessage msg,
- Throwable exception) {
- this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, exception);
+ this.logger.logIfEnabled(FQCN, Level.ERROR, READ_MARKER, msg,
+ msg.getThrowable());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da03f8b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java
index 858695a..1569ffe 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java
@@ -26,12 +26,13 @@ import java.util.Map;
public class AuditMessage implements Message {
private String message;
+ private Throwable throwable;
- public AuditMessage(String user, String ip, String op,
- Map<String, String> params, String ret){
+ private static final String MSG_PATTERN =
+ "user=%s | ip=%s | op=%s %s | ret=%s";
+
+ public AuditMessage(){
- this.message = String.format("user=%s ip=%s op=%s %s ret=%s",
- user, ip, op, params, ret);
}
@Override
@@ -51,7 +52,7 @@ public class AuditMessage implements Message {
@Override
public Throwable getThrowable() {
- return null;
+ return throwable;
}
/**
@@ -61,4 +62,70 @@ public class AuditMessage implements Message {
private void appendMessage(String customMessage) {
this.message += customMessage;
}
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public void setThrowable(Throwable throwable) {
+ this.throwable = throwable;
+ }
+
+ /**
+ * Builder class for AuditMessage.
+ */
+ public static class Builder {
+ private Throwable throwable;
+ private String user;
+ private String ip;
+ private String op;
+ private Map<String, String> params;
+ private String ret;
+
+ public Builder(){
+
+ }
+
+ public Builder setUser(String usr){
+ this.user = usr;
+ return this;
+ }
+
+ public Builder atIp(String ipAddr){
+ this.ip = ipAddr;
+ return this;
+ }
+
+ public Builder forOperation(String operation){
+ this.op = operation;
+ return this;
+ }
+
+ public Builder withParams(Map<String, String> args){
+ this.params = args;
+ return this;
+ }
+
+ public Builder withResult(String result){
+ this.ret = result;
+ return this;
+ }
+
+ public Builder withException(Throwable ex){
+ this.throwable = ex;
+ return this;
+ }
+
+ public AuditMessage build(){
+ AuditMessage auditMessage = new AuditMessage();
+ auditMessage.message = String.format(MSG_PATTERN,
+ this.user, this.ip, this.op, this.params, this.ret);
+ auditMessage.throwable = this.throwable;
+ return auditMessage;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da03f8b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditor.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditor.java
new file mode 100644
index 0000000..51c0298
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditor.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.audit;
+
+import java.util.Map;
+
+/**
+ * Interface to mark an actor as Auditor.
+ */
+public interface Auditor {
+
+ AuditMessage buildAuditMessageForSuccess(
+ AuditAction op, Map<String, String> auditMap);
+
+ AuditMessage buildAuditMessageForFailure(
+ AuditAction op, Map<String, String> auditMap, Throwable throwable);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da03f8b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
index 9c00ef7..c8284fd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
@@ -88,6 +88,12 @@ package org.apache.hadoop.ozone.audit;
* It will generate a message formatted as:
* user=xxx ip=xxx op=XXXX_XXXX {key=val, key1=val1..} ret=XXXXXX
*
+ * *** Auditor ***
+ * Interface to mark an actor class as Auditor
+ * Must be implemented by class where we want to log audit events
+ * Implementing class must override and implement methods
+ * buildAuditMessageForSuccess and buildAuditMessageForFailure.
+ *
* ****************************************************************************
* Usage
* ****************************************************************************
@@ -99,12 +105,12 @@ package org.apache.hadoop.ozone.audit;
*
* 3. Log Read/Write and Success/Failure event as needed.
* Example
- * AUDIT.logWriteSuccess(Level level, AuditMessage msg)
+ * AUDIT.logWriteSuccess(buildAuditMessageForSuccess(params))
*
- * If logging is done without specifying Level, then Level implicitly
- * defaults to INFO for xxxxSuccess() and ERROR for xxxxFailure()
- * AUDIT.logWriteSuccess(AuditMessage msg)
- * AUDIT.logWriteFailure(AuditMessage msg)
+ * 4. Log Level implicitly defaults to INFO for xxxxSuccess() and ERROR for
+ * xxxxFailure()
+ * AUDIT.logWriteSuccess(buildAuditMessageForSuccess(params))
+ * AUDIT.logWriteFailure(buildAuditMessageForSuccess(params))
*
* See sample invocations in src/test in the following class:
* org.apache.hadoop.ozone.audit.TestOzoneAuditLogger
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da03f8b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
index 6c59de6..77a6c0b 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.audit;
import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,17 +48,41 @@ public class TestOzoneAuditLogger {
private static final Map<String, String> PARAMS =
new DummyEntity().toAuditMap();
- private static final AuditMessage WRITE_FAIL_MSG = new AuditMessage("john",
- "192.168.0.1", DummyAction.CREATE_VOLUME.name(), PARAMS, FAILURE);
-
- private static final AuditMessage WRITE_SUCCESS_MSG = new AuditMessage("john",
- "192.168.0.1", DummyAction.CREATE_VOLUME.name(), PARAMS, SUCCESS);
-
- private static final AuditMessage READ_FAIL_MSG = new AuditMessage("john",
- "192.168.0.1", DummyAction.READ_VOLUME.name(), PARAMS, FAILURE);
-
- private static final AuditMessage READ_SUCCESS_MSG = new AuditMessage("john",
- "192.168.0.1", DummyAction.READ_VOLUME.name(), PARAMS, SUCCESS);
+ private static final AuditMessage WRITE_FAIL_MSG =
+ new AuditMessage.Builder()
+ .setUser("john")
+ .atIp("192.168.0.1")
+ .forOperation(DummyAction.CREATE_VOLUME.name())
+ .withParams(PARAMS)
+ .withResult(FAILURE)
+ .withException(null).build();
+
+ private static final AuditMessage WRITE_SUCCESS_MSG =
+ new AuditMessage.Builder()
+ .setUser("john")
+ .atIp("192.168.0.1")
+ .forOperation(DummyAction.CREATE_VOLUME.name())
+ .withParams(PARAMS)
+ .withResult(SUCCESS)
+ .withException(null).build();
+
+ private static final AuditMessage READ_FAIL_MSG =
+ new AuditMessage.Builder()
+ .setUser("john")
+ .atIp("192.168.0.1")
+ .forOperation(DummyAction.READ_VOLUME.name())
+ .withParams(PARAMS)
+ .withResult(FAILURE)
+ .withException(null).build();
+
+ private static final AuditMessage READ_SUCCESS_MSG =
+ new AuditMessage.Builder()
+ .setUser("john")
+ .atIp("192.168.0.1")
+ .forOperation(DummyAction.READ_VOLUME.name())
+ .withParams(PARAMS)
+ .withResult(SUCCESS)
+ .withException(null).build();
@BeforeClass
public static void setUp(){
@@ -78,24 +101,13 @@ public class TestOzoneAuditLogger {
}
/**
- * Ensures WriteSuccess events are logged @ INFO and above.
- */
- @Test
- public void logInfoWriteSuccess() throws IOException {
- AUDIT.logWriteSuccess(Level.INFO, WRITE_SUCCESS_MSG);
- String expected =
- "[INFO ] OMAudit - " + WRITE_SUCCESS_MSG.getFormattedMessage();
- verifyLog(expected);
- }
-
- /**
* Test to verify default log level is INFO when logging success events.
*/
@Test
public void verifyDefaultLogLevelForSuccess() throws IOException {
AUDIT.logWriteSuccess(WRITE_SUCCESS_MSG);
String expected =
- "[INFO ] OMAudit - " + WRITE_SUCCESS_MSG.getFormattedMessage();
+ "INFO | OMAudit | " + WRITE_SUCCESS_MSG.getFormattedMessage();
verifyLog(expected);
}
@@ -106,18 +118,7 @@ public class TestOzoneAuditLogger {
public void verifyDefaultLogLevelForFailure() throws IOException {
AUDIT.logWriteFailure(WRITE_FAIL_MSG);
String expected =
- "[ERROR] OMAudit - " + WRITE_FAIL_MSG.getFormattedMessage();
- verifyLog(expected);
- }
-
- /**
- * Test to verify WriteFailure events are logged as ERROR.
- */
- @Test
- public void logErrorWriteFailure() throws IOException {
- AUDIT.logWriteFailure(Level.ERROR, WRITE_FAIL_MSG);
- String expected =
- "[ERROR] OMAudit - " + WRITE_FAIL_MSG.getFormattedMessage();
+ "ERROR | OMAudit | " + WRITE_FAIL_MSG.getFormattedMessage();
verifyLog(expected);
}
@@ -126,20 +127,8 @@ public class TestOzoneAuditLogger {
*/
@Test
public void notLogReadEvents() throws IOException {
- AUDIT.logReadSuccess(Level.INFO, READ_SUCCESS_MSG);
- AUDIT.logReadFailure(Level.INFO, READ_FAIL_MSG);
- AUDIT.logReadFailure(Level.ERROR, READ_FAIL_MSG);
- AUDIT.logReadFailure(Level.ERROR, READ_FAIL_MSG, new Exception("test"));
- verifyNoLog();
- }
-
- /**
- * Test to ensure DEBUG level messages are not logged when INFO is enabled.
- */
- @Test
- public void notLogDebugEvents() throws IOException {
- AUDIT.logWriteSuccess(Level.DEBUG, WRITE_SUCCESS_MSG);
- AUDIT.logReadSuccess(Level.DEBUG, READ_SUCCESS_MSG);
+ AUDIT.logReadSuccess(READ_SUCCESS_MSG);
+ AUDIT.logReadFailure(READ_FAIL_MSG);
verifyNoLog();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da03f8b/hadoop-hdds/common/src/test/resources/log4j2.properties
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/resources/log4j2.properties b/hadoop-hdds/common/src/test/resources/log4j2.properties
index d60df18..cef69e1 100644
--- a/hadoop-hdds/common/src/test/resources/log4j2.properties
+++ b/hadoop-hdds/common/src/test/resources/log4j2.properties
@@ -56,13 +56,13 @@ appenders = console, audit
appender.console.type = Console
appender.console.name = STDOUT
appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%-5level] %c{1} - %msg%n
+appender.console.layout.pattern = %-5level | %c{1} | %msg%n
appender.audit.type = File
appender.audit.name = AUDITLOG
appender.audit.fileName=audit.log
appender.audit.layout.type=PatternLayout
-appender.audit.layout.pattern= [%-5level] %c{1} - %msg%n
+appender.audit.layout.pattern= %-5level | %c{1} | %msg%n
loggers=audit
logger.audit.type=AsyncLogger
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da03f8b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 6ea0fe7..7d07fcb 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.audit.AuditEventStatus;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.AuditLoggerType;
import org.apache.hadoop.ozone.audit.AuditMessage;
+import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -66,7 +67,6 @@ import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslat
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;
-import org.apache.logging.log4j.Level;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,7 +98,7 @@ import static org.apache.hadoop.util.ExitUtil.terminate;
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
public final class OzoneManager extends ServiceRuntimeInfoImpl
- implements OzoneManagerProtocol, OMMXBean {
+ implements OzoneManagerProtocol, OMMXBean, Auditor {
private static final Logger LOG =
LoggerFactory.getLogger(OzoneManager.class);
@@ -498,9 +498,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
(args == null) ? null : args.toAuditMap()));
} catch (Exception ex) {
metrics.incNumVolumeCreateFails();
- AUDIT.logWriteFailure(Level.ERROR,
+ AUDIT.logWriteFailure(
buildAuditMessageForFailure(OMAction.CREATE_VOLUME,
- (args == null) ? null : args.toAuditMap()), ex);
+ (args == null) ? null : args.toAuditMap(), ex)
+ );
throw ex;
}
}
@@ -524,7 +525,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumVolumeUpdateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.SET_OWNER,
- auditMap), ex);
+ auditMap, ex)
+ );
throw ex;
}
}
@@ -548,7 +550,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumVolumeUpdateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.SET_QUOTA,
- auditMap), ex);
+ auditMap, ex));
throw ex;
}
}
@@ -576,7 +578,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumVolumeCheckAccessFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(
- OMAction.CHECK_VOLUME_ACCESS, auditMap), ex);
+ OMAction.CHECK_VOLUME_ACCESS, auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -604,7 +606,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumVolumeInfoFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.READ_VOLUME,
- auditMap), ex);
+ auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -630,7 +632,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumVolumeDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_VOLUME,
- buildAuditMap(volume)), ex);
+ buildAuditMap(volume), ex));
throw ex;
}
}
@@ -662,7 +664,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumVolumeListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_VOLUMES,
- auditMap), ex);
+ auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -698,7 +700,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumVolumeListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_VOLUMES,
- auditMap), ex);
+ auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -724,7 +726,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumBucketCreateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.CREATE_BUCKET,
- (bucketInfo == null) ? null : bucketInfo.toAuditMap()), ex);
+ (bucketInfo == null) ? null : bucketInfo.toAuditMap(), ex));
throw ex;
}
}
@@ -750,7 +752,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumBucketListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_BUCKETS,
- auditMap), ex);
+ auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -781,7 +783,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumBucketInfoFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.READ_BUCKET,
- auditMap), ex);
+ auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -808,7 +810,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumKeyAllocateFails();
auditSuccess = false;
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.ALLOCATE_KEY,
- (args == null) ? null : args.toAuditMap()), ex);
+ (args == null) ? null : args.toAuditMap(), ex));
throw ex;
} finally {
if(auditSuccess){
@@ -832,7 +834,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumKeyCommitFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.COMMIT_KEY,
- auditMap), ex);
+ auditMap, ex));
throw ex;
}
}
@@ -851,7 +853,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumBlockAllocateCallFails();
auditSuccess = false;
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.ALLOCATE_BLOCK,
- auditMap), ex);
+ auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -878,7 +880,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumKeyLookupFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY,
- (args == null) ? null : args.toAuditMap()), ex);
+ (args == null) ? null : args.toAuditMap(), ex));
throw ex;
} finally {
if(auditSuccess){
@@ -901,7 +903,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (IOException e) {
metrics.incNumKeyRenameFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.RENAME_KEY,
- auditMap), e);
+ auditMap, e));
throw e;
}
}
@@ -922,7 +924,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumKeyDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_KEY,
- (args == null) ? null : args.toAuditMap()), ex);
+ (args == null) ? null : args.toAuditMap(), ex));
throw ex;
}
}
@@ -944,7 +946,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metrics.incNumKeyListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_KEYS,
- auditMap), ex);
+ auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
@@ -971,7 +973,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumBucketUpdateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.UPDATE_BUCKET,
- (args == null) ? null : args.toAuditMap()), ex);
+ (args == null) ? null : args.toAuditMap(), ex));
throw ex;
}
}
@@ -994,7 +996,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
} catch (Exception ex) {
metrics.incNumBucketDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_BUCKET,
- auditMap), ex);
+ auditMap, ex));
throw ex;
}
}
@@ -1005,26 +1007,34 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return auditMap;
}
- // TODO: Temporary method until AuditMessage is simplified
- private AuditMessage buildAuditMessageForSuccess(AuditAction op,
+ @Override
+ public AuditMessage buildAuditMessageForSuccess(AuditAction op,
Map<String, String> auditMap) {
- return new AuditMessage(
- (Server.getRemoteUser() == null) ? null :
- Server.getRemoteUser().getUserName(),
- (Server.getRemoteIp() == null) ? null :
- Server.getRemoteIp().getHostAddress(), op.toString(), auditMap,
- AuditEventStatus.SUCCESS.toString());
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.SUCCESS.toString())
+ .withException(null)
+ .build();
}
- // TODO: Temporary method until AuditMessage is simplified
- private AuditMessage buildAuditMessageForFailure(AuditAction op,
- Map<String, String> auditMap) {
- return new AuditMessage(
- (Server.getRemoteUser() == null) ? null :
- Server.getRemoteUser().getUserName(),
- (Server.getRemoteIp() == null) ? null :
- Server.getRemoteIp().getHostAddress(), op.toString(), auditMap,
- AuditEventStatus.FAILURE.toString());
+ @Override
+ public AuditMessage buildAuditMessageForFailure(AuditAction op,
+ Map<String, String> auditMap, Throwable throwable) {
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.FAILURE.toString())
+ .withException(throwable)
+ .build();
}
private void registerMXBean() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDFS-13932. [JDK 11] Casts to
BlockStoragePolicy[] in unit tests raise ClassCastExceptions. Contributed by
Andrew Purtell.
Posted by in...@apache.org.
HDFS-13932. [JDK 11] Casts to BlockStoragePolicy[] in unit tests raise ClassCastExceptions. Contributed by Andrew Purtell.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9c02216
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9c02216
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9c02216
Branch: refs/heads/HDFS-13532
Commit: f9c0221623ee5849453f7ec9021d1697e9677805
Parents: 6b988d8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Sep 27 11:44:42 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Sep 27 11:44:42 2018 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java | 7 +------
.../test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java | 8 ++------
2 files changed, 3 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9c02216/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index 0ffb62c..c7dc6f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
@@ -1029,13 +1028,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
Path path = new Path(getProxiedFSTestDir(), "policy.txt");
FileSystem httpfs = getHttpFSFileSystem();
// test getAllStoragePolicies
- BlockStoragePolicy[] dfsPolicies = (BlockStoragePolicy[]) fs
- .getAllStoragePolicies().toArray();
- BlockStoragePolicy[] httpPolicies = (BlockStoragePolicy[]) httpfs
- .getAllStoragePolicies().toArray();
Assert.assertArrayEquals(
"Policy array returned from the DFS and HttpFS should be equals",
- dfsPolicies, httpPolicies);
+ fs.getAllStoragePolicies().toArray(), httpfs.getAllStoragePolicies().toArray());
// test get/set/unset policies
DFSTestUtil.createFile(fs, path, 0, (short) 1, 0L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9c02216/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index a766de6..d7ed46b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -81,7 +81,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.TestDFSClientRetries;
import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -1405,11 +1404,8 @@ public class TestWebHDFS {
conf, WebHdfsConstants.WEBHDFS_SCHEME);
// test getAllStoragePolicies
- BlockStoragePolicy[] dfsPolicies = (BlockStoragePolicy[]) dfs
- .getAllStoragePolicies().toArray();
- BlockStoragePolicy[] webHdfsPolicies = (BlockStoragePolicy[]) webHdfs
- .getAllStoragePolicies().toArray();
- Assert.assertTrue(Arrays.equals(dfsPolicies, webHdfsPolicies));
+ Assert.assertTrue(Arrays.equals(dfs.getAllStoragePolicies().toArray(),
+ webHdfs.getAllStoragePolicies().toArray()));
// test get/set/unset policies
DFSTestUtil.createFile(dfs, path, 0, (short) 1, 0L);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: Changed version in trunk to
3.3.0-SNAPSHOT.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
index cb0d6e2..81a8f4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
@@ -22,13 +22,13 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
<name>Apache Hadoop YARN TimelineService HBase Common</name>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
index 79fc861..771580b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
@@ -22,13 +22,13 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-1</artifactId>
<name>Apache Hadoop YARN TimelineService HBase Server 1.2</name>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
index c9fe5d0..4fde40c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
@@ -22,13 +22,13 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
<name>Apache Hadoop YARN TimelineService HBase Server 2.0</name>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
index dd2928c..816bf37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN TimelineService HBase Servers</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
index 7e5a803..b3f974a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN TimelineService HBase Backend</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 35d5c94..55cb3b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Service</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index 0d1b92b..aa9f753 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Web Proxy</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index de4484c..2f51023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Server</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
index e9ecb65..6a75235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-site</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Site</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index a0e530a..a80b580 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@ -20,11 +20,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-ui</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN UI</name>
<packaging>${packagingType}</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 6110476..e8a7dad 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -16,11 +16,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-yarn</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Hadoop YARN</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 4593441..0882027 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-yarn-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Hadoop YARN Project</name>
<url>http://hadoop.apache.org/yarn/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5780a79..b2606dd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Main</description>
<name>Apache Hadoop Main</name>
<packaging>pom</packaging>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDDS-562. Create acceptance test
to test aws cli with the s3 gateway. Contributed by Elek Marton.
Posted by in...@apache.org.
HDDS-562. Create acceptance test to test aws cli with the s3 gateway. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d082193
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d082193
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d082193
Branch: refs/heads/HDFS-13532
Commit: 7d082193d2c55b89210b277cd9a0dc2f4e590bee
Parents: d0ee6fb
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Oct 1 15:04:20 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Oct 1 15:04:20 2018 -0700
----------------------------------------------------------------------
.../dist/src/main/smoketest/s3/awscli.robot | 63 ++++++++++++++++++++
hadoop-ozone/dist/src/main/smoketest/test.sh | 3 +-
2 files changed, 65 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d082193/hadoop-ozone/dist/src/main/smoketest/s3/awscli.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/awscli.robot b/hadoop-ozone/dist/src/main/smoketest/s3/awscli.robot
new file mode 100644
index 0000000..b26ad91
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/awscli.robot
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 gateway test with aws cli
+Library OperatingSystem
+Library String
+Resource ../commonlib.robot
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g:9878
+
+*** Keywords ***
+Execute AWSCli
+ [Arguments] ${command}
+ ${output} = Execute aws s3 --endpoint-url ${ENDPOINT_URL}/${VOLUME} ${command}
+ [return] ${output}
+
+*** Test Cases ***
+
+Create volume and bucket for the tests
+ ${postfix} = Generate Random String 5 [NUMBERS]
+ Set Suite Variable ${BUCKET} bucket-${postfix}
+ Set Suite Variable ${VOLUME} vol-${postfix}
+ Log Testing s3 commands in /${VOLUME}/${BUCKET}
+ ${result} = Execute ozone sh volume create /${VOLUME} --user hadoop
+ ${result} = Execute ozone sh bucket create /${VOLUME}/${BUCKET}
+
+Install aws s3 cli
+ Execute sudo apt-get install -y awscli
+ Set Environment Variable AWS_ACCESS_KEY_ID ANYID
+ Set Environment Variable AWS_SECRET_ACCESS_KEY ANYKEY
+
+File upload and directory list
+ Execute date > /tmp/testfile
+ ${result} = Execute AWSCli cp /tmp/testfile s3://${BUCKET}
+ Should contain ${result} upload
+ ${result} = Execute AWSCli cp /tmp/testfile s3://${BUCKET}/dir1/dir2/file
+ Should contain ${result} upload
+ ${result} = Execute AWSCli ls s3://${BUCKET}
+ Should contain ${result} testfile
+ Should contain ${result} dir1
+ Should not contain ${result} dir2
+ ${result} = Execute AWSCli ls s3://${BUCKET}/dir1/
+ Should not contain ${result} testfile
+ Should not contain ${result} dir1
+ Should contain ${result} dir2
+ ${result} = Execute AWSCli ls s3://${BUCKET}/dir1/dir2/
+ Should not contain ${result} testfile
+ Should not contain ${result} dir1
+ Should contain ${result} file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d082193/hadoop-ozone/dist/src/main/smoketest/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh
index 534bbb7..b32955a 100755
--- a/hadoop-ozone/dist/src/main/smoketest/test.sh
+++ b/hadoop-ozone/dist/src/main/smoketest/test.sh
@@ -95,7 +95,8 @@ if [ "$RUN_ALL" = true ]; then
execute_tests ozone "${DEFAULT_TESTS[@]}"
TESTS=("ozonefs")
execute_tests ozonefs "${TESTS[@]}"
-
+ TESTS=("s3")
+ execute_tests ozones3 "${TESTS[@]}"
else
execute_tests "$DOCKERENV" "${POSITIONAL[@]}"
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-8804. resourceLimits may be
wrongly calculated when leaf-queue is blocked in cluster with 3+ level
queues. Contributed by Tao Yang
Posted by in...@apache.org.
YARN-8804. resourceLimits may be wrongly calculated when leaf-queue is blocked in cluster with 3+ level queues. Contributed by Tao Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b988d82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b988d82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b988d82
Branch: refs/heads/HDFS-13532
Commit: 6b988d821e62d29c118e10a7213583b92c302baf
Parents: 913f87d
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 26 14:43:00 2018 -0700
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 26 14:43:00 2018 -0700
----------------------------------------------------------------------
.../scheduler/ResourceLimits.java | 24 ++++++
.../scheduler/capacity/ParentQueue.java | 16 ++--
.../capacity/TestContainerAllocation.java | 78 ++++++++++++++++++++
3 files changed, 112 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b988d82/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
index 721eb36..820d2fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
@@ -38,6 +38,9 @@ public class ResourceLimits {
// containers.
private volatile Resource headroom;
+ // How much resource should be reserved for high-priority blocked queues
+ private Resource blockedHeadroom;
+
private boolean allowPreempt = false;
public ResourceLimits(Resource limit) {
@@ -81,4 +84,25 @@ public class ResourceLimits {
public void setIsAllowPreemption(boolean allowPreempt) {
this.allowPreempt = allowPreempt;
}
+
+ public void addBlockedHeadroom(Resource resource) {
+ if (blockedHeadroom == null) {
+ blockedHeadroom = Resource.newInstance(0, 0);
+ }
+ Resources.addTo(blockedHeadroom, resource);
+ }
+
+ public Resource getBlockedHeadroom() {
+ if (blockedHeadroom == null) {
+ return Resources.none();
+ }
+ return blockedHeadroom;
+ }
+
+ public Resource getNetLimit() {
+ if (blockedHeadroom != null) {
+ return Resources.subtract(limit, blockedHeadroom);
+ }
+ return limit;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b988d82/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 80549ca..e32130f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -776,7 +776,6 @@ public class ParentQueue extends AbstractCSQueue {
SchedulingMode schedulingMode) {
CSAssignment assignment = CSAssignment.NULL_ASSIGNMENT;
- Resource parentLimits = limits.getLimit();
printChildQueues();
// Try to assign to most 'under-served' sub-queue
@@ -790,7 +789,7 @@ public class ParentQueue extends AbstractCSQueue {
// Get ResourceLimits of child queue before assign containers
ResourceLimits childLimits =
- getResourceLimitsOfChild(childQueue, cluster, parentLimits,
+ getResourceLimitsOfChild(childQueue, cluster, limits.getNetLimit(),
candidates.getPartition());
CSAssignment childAssignment = childQueue.assignContainers(cluster,
@@ -812,16 +811,21 @@ public class ParentQueue extends AbstractCSQueue {
CSAssignment.SkippedType.QUEUE_LIMIT) {
assignment = childAssignment;
}
+ Resource blockedHeadroom = null;
+ if (childQueue instanceof LeafQueue) {
+ blockedHeadroom = childLimits.getHeadroom();
+ } else {
+ blockedHeadroom = childLimits.getBlockedHeadroom();
+ }
Resource resourceToSubtract = Resources.max(resourceCalculator,
- cluster, childLimits.getHeadroom(), Resources.none());
+ cluster, blockedHeadroom, Resources.none());
+ limits.addBlockedHeadroom(resourceToSubtract);
if(LOG.isDebugEnabled()) {
- LOG.debug("Decrease parentLimits " + parentLimits +
+ LOG.debug("Decrease parentLimits " + limits.getLimit() +
" for " + this.getQueueName() + " by " +
resourceToSubtract + " as childQueue=" +
childQueue.getQueueName() + " is blocked");
}
- parentLimits = Resources.subtract(parentLimits,
- resourceToSubtract);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b988d82/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 3d028ee..f1b4444 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -1052,4 +1052,82 @@ public class TestContainerAllocation {
rm1.close();
}
+
+ @Test(timeout = 60000)
+ public void testAllocationCannotBeBlockedWhenFormerQueueReachedItsLimit()
+ throws Exception {
+ /**
+ * Queue structure:
+ * <pre>
+ * Root
+ * / | \
+ * a b c
+ * 10 20 70
+ * | \
+ * c1 c2
+ * 10(max=10) 90
+ * </pre>
+ * Test case:
+ * Create a cluster with two nodes whose node resource both are
+ * <10GB, 10core>, create queues as above, among them max-capacity of "c1"
+ * is 10 and others are all 100, so that max-capacity of queue "c1" is
+ * <2GB, 2core>,
+ * submit app1 to queue "c1" and launch am1(resource=<1GB, 1 core>) on nm1,
+ * submit app2 to queue "b" and launch am2(resource=<1GB, 1 core>) on nm1,
+ * app1 and app2 both ask one <2GB, 1core> containers
+ *
+ * Now queue "c" has lower capacity percentage than queue "b", the
+ * allocation sequence will be "a" -> "c" -> "b", queue "c1" has reached
+ * queue limit so that requests of app1 should be pending
+ *
+ * After nm1 do 1 heartbeat, scheduler should allocate one container for
+ * app2 on nm1.
+ */
+ CapacitySchedulerConfiguration newConf =
+ (CapacitySchedulerConfiguration) TestUtils
+ .getConfigurationWithMultipleQueues(conf);
+ newConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".c",
+ new String[] { "c1", "c2" });
+ newConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c1", 10);
+ newConf
+ .setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c1", 10);
+ newConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 90);
+ newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+
+ MockRM rm1 = new MockRM(newConf);
+
+ RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
+ nodeLabelsManager.init(newConf);
+ rm1.getRMContext().setNodeLabelManager(nodeLabelsManager);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB);
+
+ // launch an app to queue "c1", AM container should be launched on nm1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "c1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // launch another app to queue "b", AM container should be launched on nm1
+ RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ am1.allocate("*", 2 * GB, 1, new ArrayList<ContainerId>());
+ am2.allocate("*", 2 * GB, 1, new ArrayList<ContainerId>());
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+ FiCaSchedulerApp schedulerApp2 =
+ cs.getApplicationAttempt(am2.getApplicationAttemptId());
+
+ // Do nm1 heartbeats 1 times, will allocate a container on nm1 for app2
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ rm1.drainEvents();
+ Assert.assertEquals(1, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(2, schedulerApp2.getLiveContainers().size());
+
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-13840. RBW Blocks which are
having less GS should be added to Corrupt. Contributed by Brahma Reddy
Battula
Posted by in...@apache.org.
HDFS-13840. RBW Blocks which are having less GS should be added to Corrupt. Contributed by Brahma Reddy Battula
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0ff8e2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0ff8e2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0ff8e2c
Branch: refs/heads/HDFS-13532
Commit: e0ff8e2c10f38298b81624df1e62743ac171f18d
Parents: 6275b42
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Sep 26 23:44:16 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Sep 26 23:44:16 2018 +0530
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 19 ++++++++--
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 5 +++
.../TestRBWBlockInvalidation.java | 38 ++++++++++++++++++++
.../server/namenode/TestAddStripedBlocks.java | 4 +--
4 files changed, 61 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ff8e2c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 675221a..5e14247 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3086,10 +3086,20 @@ public class BlockManager implements BlockStatsMXBean {
}
case RBW:
case RWR:
+ final long reportedGS = reported.getGenerationStamp();
if (!storedBlock.isComplete()) {
+ //When DN report lesser GS than the storedBlock then mark it is corrupt,
+ //As already valid replica will be present.
+ if (storedBlock.getGenerationStamp() > reported.getGenerationStamp()) {
+ return new BlockToMarkCorrupt(new Block(reported), storedBlock,
+ reportedGS,
+ "reported " + reportedState + " replica with genstamp "
+ + reportedGS
+ + " does not match Stored block's genstamp in block map "
+ + storedBlock.getGenerationStamp(), Reason.GENSTAMP_MISMATCH);
+ }
return null; // not corrupt
} else if (storedBlock.getGenerationStamp() != reported.getGenerationStamp()) {
- final long reportedGS = reported.getGenerationStamp();
return new BlockToMarkCorrupt(new Block(reported), storedBlock, reportedGS,
"reported " + reportedState + " replica with genstamp " + reportedGS
+ " does not match COMPLETE block's genstamp in block map "
@@ -3149,8 +3159,11 @@ public class BlockManager implements BlockStatsMXBean {
block.getUnderConstructionFeature().addReplicaIfNotPresent(
storageInfo, ucBlock.reportedBlock, ucBlock.reportedState);
- if (ucBlock.reportedState == ReplicaState.FINALIZED &&
- (block.findStorageInfo(storageInfo) < 0)) {
+ // Add replica if appropriate. If the replica was previously corrupt
+ // but now okay, it might need to be updated.
+ if (ucBlock.reportedState == ReplicaState.FINALIZED && (
+ block.findStorageInfo(storageInfo) < 0) || corruptReplicas
+ .isReplicaCorrupt(block, storageInfo.getDatanodeDescriptor())) {
addStoredBlock(block, ucBlock.reportedBlock, storageInfo, null, true);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ff8e2c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 11265b8..29807ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -560,6 +560,11 @@ public class MiniDFSCluster implements AutoCloseable {
public void setDnArgs(String ... args) {
dnArgs = args;
}
+
+ public DataNode getDatanode() {
+ return datanode;
+ }
+
}
private Configuration conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ff8e2c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
index 40f54cb..9c48eeb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestRBWBlockInvalidation.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import java.io.Closeable;
import java.io.IOException;
@@ -27,6 +28,7 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -245,6 +247,42 @@ public class TestRBWBlockInvalidation {
}
+ @Test
+ public void testRWRShouldNotAddedOnDNRestart() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.set("dfs.client.block.write.replace-datanode-on-failure.enable",
+ "false");
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(2).build()) {
+ Path path = new Path("/testRBW");
+ FSDataOutputStream out = cluster.getFileSystem().create(path, (short) 2);
+ out.writeBytes("old gs data\n");
+ out.hflush();
+ // stop one datanode
+ DataNodeProperties dnProp = cluster.stopDataNode(0);
+ String dnAddress = dnProp.getDatanode().getXferAddress().toString();
+ if (dnAddress.startsWith("/")) {
+ dnAddress = dnAddress.substring(1);
+ }
+ //Write some more data after DN stopped.
+ out.writeBytes("old gs data\n");
+ out.hflush();
+ cluster.restartDataNode(dnProp, true);
+ // wait till the block report comes
+ Thread.sleep(3000);
+ // check the block locations, this should not contain restarted datanode
+ BlockLocation[] locations = cluster.getFileSystem()
+ .getFileBlockLocations(path, 0, Long.MAX_VALUE);
+ String[] names = locations[0].getNames();
+ for (String node : names) {
+ if (node.equals(dnAddress)) {
+ fail("Old GS DN should not be present in latest block locations.");
+ }
+ }
+ out.close();
+ }
+ }
+
private void waitForNumTotalBlocks(final MiniDFSCluster cluster,
final int numTotalBlocks) throws Exception {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ff8e2c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
index ec13b44..9374c04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
@@ -337,8 +337,8 @@ public class TestAddStripedBlocks {
int i = groupSize - 1;
for (DataNode dn : cluster.getDataNodes()) {
String storageID = storageIDs.get(i);
- final Block block = new Block(lastBlock.getBlockId() + i--,
- lastBlock.getGenerationStamp(), 0);
+ final Block block = new Block(lastBlock.getBlockId() + i--, 0,
+ lastBlock.getGenerationStamp());
DatanodeStorage storage = new DatanodeStorage(storageID);
List<ReplicaBeingWritten> blocks = new ArrayList<>();
ReplicaBeingWritten replica = new ReplicaBeingWritten(block, null, null,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-455. Ozone genconf tool must
use picocli. Contributed by Dinesh Chitlangia.
Posted by in...@apache.org.
HDDS-455. Ozone genconf tool must use picocli. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35c7351f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35c7351f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35c7351f
Branch: refs/heads/HDFS-13532
Commit: 35c7351f4bf2d17439db9adc571583d89a267f35
Parents: 19ad5be
Author: Hanisha Koneru <ha...@apache.org>
Authored: Fri Sep 28 13:06:50 2018 -0700
Committer: Hanisha Koneru <ha...@apache.org>
Committed: Fri Sep 28 13:06:50 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/Settings.md | 2 +-
.../GenerateOzoneRequiredConfigurations.java | 130 ++++++---------
...TestGenerateOzoneRequiredConfigurations.java | 167 +++++++++++++------
3 files changed, 167 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c7351f/hadoop-ozone/docs/content/Settings.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/Settings.md b/hadoop-ozone/docs/content/Settings.md
index b2d30e5..41ab04a 100644
--- a/hadoop-ozone/docs/content/Settings.md
+++ b/hadoop-ozone/docs/content/Settings.md
@@ -53,7 +53,7 @@ following command. This will generate a template called ```ozone-site.xml``` at
the specified path (directory).
{{< highlight bash >}}
-ozone genconf -output <path>
+ozone genconf <path>
{{< /highlight >}}
Let us look at the settings inside the generated file (ozone-site.xml) and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c7351f/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java
index 688b735..b58cfc2 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java
@@ -18,7 +18,12 @@
package org.apache.hadoop.ozone.genconf;
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Parameters;
+import picocli.CommandLine.PicocliException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
@@ -30,112 +35,58 @@ import java.nio.file.InvalidPathException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
-
/**
* GenerateOzoneRequiredConfigurations - A tool to generate ozone-site.xml<br>
* This tool generates an ozone-site.xml with minimally required configs.
* This tool can be invoked as follows:<br>
* <ul>
- * <li>ozone genconf -output <Path to output file></li>
- * <li>ozone genconf -help</li>
+ * <li>ozone genconf <Path to output file></li>
+ * <li>ozone genconf --help</li>
+ * <li>ozone genconf -h</li>
* </ul>
*/
-public final class GenerateOzoneRequiredConfigurations {
-
- private static final String OUTPUT = "-output";
- private static final String HELP = "-help";
- private static final String USAGE = "Usage: \nozone genconf "
- + OUTPUT + " <Path to output file> \n"
- + "ozone genconf "
- + HELP;
- private static final int SUCCESS = 0;
- private static final int FAILURE = 1;
+@Command(
+ name = "ozone genconf",
+ description = "Tool to generate template ozone-site.xml",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true)
+public final class GenerateOzoneRequiredConfigurations extends GenericCli {
- private GenerateOzoneRequiredConfigurations() {
+ @Parameters(arity = "1..1",
+ description = "Directory path where ozone-site file should be generated.")
+ private String path;
- }
/**
* Entry point for using genconf tool.
*
* @param args
- * @throws JAXBException
- */
- public static void main(String[] args) {
-
- try {
- if (args.length == 0) {
- System.out.println(USAGE);
- System.exit(1);
- }
-
- switch (args[0]) {
- case OUTPUT:
- if (args.length > 1) {
- int result = generateConfigurations(args[1]);
- } else {
- System.out.println("Path to output file is mandatory");
- System.out.println(USAGE);
- System.exit(1);
- }
- break;
-
- case HELP:
- System.out.println(USAGE);
- System.exit(0);
- break;
-
- default:
- System.out.println(USAGE);
- System.exit(1);
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- /**
- * Check if the path is valid directory.
*
- * @param path
- * @return true, if path is valid directory, else return false
*/
- public static boolean isValidPath(String path) {
- try {
- return Files.isDirectory(Paths.get(path));
- } catch (InvalidPathException | NullPointerException ex) {
- return false;
- }
+ public static void main(String[] args) throws Exception {
+ new GenerateOzoneRequiredConfigurations().run(args);
}
- /**
- * Check if user has permission to write in the specified path.
- *
- * @param path
- * @return true, if the user has permission to write, else returns false
- */
- public static boolean canWrite(String path) {
- File file = new File(path);
- return file.canWrite();
+ @Override
+ public Void call() throws Exception {
+ generateConfigurations(path);
+ return null;
}
/**
* Generate ozone-site.xml at specified path.
- *
* @param path
- * @return SUCCESS(0) if file can be generated, else returns FAILURE(1)
+ * @throws PicocliException
* @throws JAXBException
*/
- public static int generateConfigurations(String path) throws JAXBException {
+ public static void generateConfigurations(String path) throws
+ PicocliException, JAXBException {
if (!isValidPath(path)) {
- System.out.println("Invalid directory path.");
- return FAILURE;
+ throw new PicocliException("Invalid directory path.");
}
if (!canWrite(path)) {
- System.out.println("Insufficient permission.");
- return FAILURE;
+ throw new PicocliException("Insufficient permission.");
}
OzoneConfiguration oc = new OzoneConfiguration();
@@ -168,7 +119,30 @@ public final class GenerateOzoneRequiredConfigurations {
m.marshal(requiredConfig, new File(path, "ozone-site.xml"));
System.out.println("ozone-site.xml has been generated at " + path);
+ }
+
+ /**
+ * Check if the path is valid directory.
+ *
+ * @param path
+ * @return true, if path is valid directory, else return false
+ */
+ public static boolean isValidPath(String path) {
+ try {
+ return Files.isDirectory(Paths.get(path));
+ } catch (InvalidPathException | NullPointerException ex) {
+ return Boolean.FALSE;
+ }
+ }
- return SUCCESS;
+ /**
+ * Check if user has permission to write in the specified path.
+ *
+ * @param path
+ * @return true, if the user has permission to write, else returns false
+ */
+ public static boolean canWrite(String path) {
+ File file = new File(path);
+ return file.canWrite();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c7351f/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
index 8c75ebb..0fab185 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
@@ -21,23 +21,40 @@ package org.apache.hadoop.ozone.genconf;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.test.GenericTestUtils;
-import org.hamcrest.CoreMatchers;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.ExecutionException;
+import picocli.CommandLine.IExceptionHandler2;
+import picocli.CommandLine.ParseResult;
+import picocli.CommandLine.ParameterException;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
-
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
/**
* Tests GenerateOzoneRequiredConfigurations.
*/
public class TestGenerateOzoneRequiredConfigurations {
private static File outputBaseDir;
+ private static GenerateOzoneRequiredConfigurations genconfTool;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestGenerateOzoneRequiredConfigurations.class);
+ private final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream err = new ByteArrayOutputStream();
+ private static final PrintStream OLD_OUT = System.out;
+ private static final PrintStream OLD_ERR = System.err;
/**
* Creates output directory which will be used by the test-cases.
* If a test-case needs a separate directory, it has to create a random
@@ -49,6 +66,24 @@ public class TestGenerateOzoneRequiredConfigurations {
public static void init() throws Exception {
outputBaseDir = GenericTestUtils.getTestDir();
FileUtils.forceMkdir(outputBaseDir);
+ genconfTool = new GenerateOzoneRequiredConfigurations();
+ }
+
+ @Before
+ public void setup() throws Exception {
+ System.setOut(new PrintStream(out));
+ System.setErr(new PrintStream(err));
+ }
+
+ @After
+ public void reset() {
+ // reset stream after each unit test
+ out.reset();
+ err.reset();
+
+ // restore system streams
+ System.setOut(OLD_OUT);
+ System.setErr(OLD_ERR);
}
/**
@@ -59,6 +94,57 @@ public class TestGenerateOzoneRequiredConfigurations {
FileUtils.deleteDirectory(outputBaseDir);
}
+ private void execute(String[] args, String msg) {
+ List<String> arguments = new ArrayList(Arrays.asList(args));
+ LOG.info("Executing shell command with args {}", arguments);
+ CommandLine cmd = genconfTool.getCmd();
+
+ IExceptionHandler2<List<Object>> exceptionHandler =
+ new IExceptionHandler2<List<Object>>() {
+ @Override
+ public List<Object> handleParseException(ParameterException ex,
+ String[] args) {
+ throw ex;
+ }
+
+ @Override
+ public List<Object> handleExecutionException(ExecutionException ex,
+ ParseResult parseResult) {
+ throw ex;
+ }
+ };
+ cmd.parseWithHandlers(new CommandLine.RunLast(),
+ exceptionHandler, args);
+ Assert.assertTrue(out.toString().contains(msg));
+ }
+
+ private void executeWithException(String[] args, String msg) {
+ List<String> arguments = new ArrayList(Arrays.asList(args));
+ LOG.info("Executing shell command with args {}", arguments);
+ CommandLine cmd = genconfTool.getCmd();
+
+ IExceptionHandler2<List<Object>> exceptionHandler =
+ new IExceptionHandler2<List<Object>>() {
+ @Override
+ public List<Object> handleParseException(ParameterException ex,
+ String[] args) {
+ throw ex;
+ }
+
+ @Override
+ public List<Object> handleExecutionException(ExecutionException ex,
+ ParseResult parseResult) {
+ throw ex;
+ }
+ };
+ try{
+ cmd.parseWithHandlers(new CommandLine.RunLast(),
+ exceptionHandler, args);
+ }catch(Exception ex){
+ Assert.assertTrue(ex.getMessage().contains(msg));
+ }
+ }
+
/**
* Tests a valid path and generates ozone-site.xml by calling
* {@code GenerateOzoneRequiredConfigurations#generateConfigurations}.
@@ -68,79 +154,54 @@ public class TestGenerateOzoneRequiredConfigurations {
@Test
public void testGenerateConfigurations() throws Exception {
File tempPath = getRandomTempDir();
- String[] args = new String[]{"-output", tempPath.getAbsolutePath()};
-
- Assert.assertEquals("Path is valid",
- true, GenerateOzoneRequiredConfigurations.isValidPath(args[1]));
-
- Assert.assertEquals("Permission is valid",
- true, GenerateOzoneRequiredConfigurations.canWrite(args[1]));
-
- Assert.assertEquals("Config file generated",
- 0, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1]));
+ String[] args = new String[]{tempPath.getAbsolutePath()};
+ execute(args, "ozone-site.xml has been generated at " +
+ tempPath.getAbsolutePath());
}
/**
- * Tests ozone-site.xml generation by calling
- * {@code GenerateOzoneRequiredConfigurations#main}.
- *
+ * Test to avoid generating ozone-site.xml when insufficient permission.
* @throws Exception
*/
@Test
- public void testGenerateConfigurationsThroughMainMethod() throws Exception {
+ public void genconfFailureByInsufficientPermissions() throws Exception {
File tempPath = getRandomTempDir();
- String[] args = new String[]{"-output", tempPath.getAbsolutePath()};
- ByteArrayOutputStream outContent = new ByteArrayOutputStream();
- PrintStream oldStream = System.out;
- try (PrintStream ps = new PrintStream(outContent)) {
- System.setOut(ps);
- GenerateOzoneRequiredConfigurations.main(args);
- Assert.assertThat(outContent.toString(), CoreMatchers.containsString(
- "ozone-site.xml has been generated at"));
- System.setOut(oldStream);
- }
+ tempPath.setReadOnly();
+ String[] args = new String[]{tempPath.getAbsolutePath()};
+ executeWithException(args, "Insufficient permission.");
}
/**
- * Test to avoid generating ozone-site.xml when invalid permission.
+ * Test to avoid generating ozone-site.xml when invalid path.
* @throws Exception
*/
@Test
- public void generateConfigurationsFailure() throws Exception {
+ public void genconfFailureByInvalidPath() throws Exception {
File tempPath = getRandomTempDir();
- tempPath.setReadOnly();
- String[] args = new String[]{"-output", tempPath.getAbsolutePath()};
- GenerateOzoneRequiredConfigurations.main(args);
-
- Assert.assertEquals("Path is valid",
- true, GenerateOzoneRequiredConfigurations.isValidPath(args[1]));
-
- Assert.assertEquals("Invalid permission",
- false, GenerateOzoneRequiredConfigurations.canWrite(args[1]));
-
- Assert.assertEquals("Config file not generated",
- 1, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1]));
- tempPath.setWritable(true);
+ String[] args = new String[]{"invalid-path"};
+ executeWithException(args, "Invalid directory path.");
}
/**
- * Test to avoid generating ozone-site.xml when invalid permission.
+ * Test to avoid generating ozone-site.xml when path not specified.
* @throws Exception
*/
@Test
- public void generateConfigurationsFailureForInvalidPath() throws Exception {
+ public void genconfPathNotSpecified() throws Exception {
File tempPath = getRandomTempDir();
- tempPath.setReadOnly();
- String[] args = new String[]{"-output",
- tempPath.getAbsolutePath() + "/ozone-site.xml"};
- GenerateOzoneRequiredConfigurations.main(args);
-
- Assert.assertEquals("Path is invalid", false,
- GenerateOzoneRequiredConfigurations.isValidPath(args[1]));
+ String[] args = new String[]{};
+ executeWithException(args, "Missing required parameter: <path>");
+ }
- Assert.assertEquals("Config file not generated", 1,
- GenerateOzoneRequiredConfigurations.generateConfigurations(args[1]));
- tempPath.setWritable(true);
+ /**
+ * Test to check help message.
+ * @throws Exception
+ */
+ @Test
+ public void genconfHelp() throws Exception {
+ File tempPath = getRandomTempDir();
+ String[] args = new String[]{"--help"};
+ execute(args, "Usage: ozone genconf [-hV] [--verbose]");
}
private File getRandomTempDir() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-15739. ABFS: remove unused
maven dependencies and add used undeclared dependencies. Contributed by Da
Zhou.
Posted by in...@apache.org.
HADOOP-15739. ABFS: remove unused maven dependencies and add used undeclared dependencies.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4abf020
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4abf020
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4abf020
Branch: refs/heads/HDFS-13532
Commit: a4abf02028ab4d8558cd393cf65228c6aa720827
Parents: d5da992
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 25 20:58:32 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 25 20:58:32 2018 +0100
----------------------------------------------------------------------
hadoop-tools/hadoop-azure/pom.xml | 40 +++++++++++++++++++++++-----------
1 file changed, 27 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4abf020/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 52f0bae..5de9875 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -169,19 +169,6 @@
</dependency>
<dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <scope>compile</scope>
- <!-- we have a dependency on a lower version -->
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -210,6 +197,21 @@
<scope>compile</scope>
</dependency>
+ <!--com.fasterxml.jackson is used by WASB, not ABFS-->
+ <!--transitive dependency from Azure SDK-->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!--transitive dependency from hadoop-common-->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<!-- dependencies use for test only -->
<dependency>
<groupId>junit</groupId>
@@ -244,6 +246,18 @@
</dependency>
<dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-8270 Adding JMX Metrics for
Timeline Collector and Reader. Contributed by Sushil Ks.
Posted by in...@apache.org.
YARN-8270 Adding JMX Metrics for Timeline Collector and Reader. Contributed by Sushil Ks.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90e2e493
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90e2e493
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90e2e493
Branch: refs/heads/HDFS-13532
Commit: 90e2e493b3dc8be54f655b957b98a4bc0e003684
Parents: b237a0d
Author: Vrushali C <vr...@apache.org>
Authored: Thu Sep 27 15:53:39 2018 -0700
Committer: Vrushali C <vr...@apache.org>
Committed: Thu Sep 27 15:53:39 2018 -0700
----------------------------------------------------------------------
.../collector/TimelineCollectorWebService.java | 16 +-
.../PerNodeAggTimelineCollectorMetrics.java | 117 +++++++++++++
.../metrics/TimelineReaderMetrics.java | 113 +++++++++++++
.../timelineservice/metrics/package-info.java | 28 ++++
.../reader/TimelineReaderWebServices.java | 166 +++++++++++++------
.../TestPerNodeAggTimelineCollectorMetrics.java | 56 +++++++
.../reader/TestTimelineReaderMetrics.java | 56 +++++++
7 files changed, 500 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e2e493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index b33a0f0..5111da8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
@@ -54,6 +55,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.UserEntity;
+import org.apache.hadoop.yarn.server.timelineservice.metrics.PerNodeAggTimelineCollectorMetrics;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -78,6 +80,8 @@ public class TimelineCollectorWebService {
LoggerFactory.getLogger(TimelineCollectorWebService.class);
private @Context ServletContext context;
+ private static final PerNodeAggTimelineCollectorMetrics METRICS =
+ PerNodeAggTimelineCollectorMetrics.getInstance();
/**
* Gives information about timeline collector.
@@ -152,12 +156,15 @@ public class TimelineCollectorWebService {
TimelineEntities entities) {
init(res);
UserGroupInformation callerUgi = getUser(req);
+ boolean isAsync = async != null && async.trim().equalsIgnoreCase("true");
if (callerUgi == null) {
String msg = "The owner of the posted timeline entities is not set";
LOG.error(msg);
throw new ForbiddenException(msg);
}
+ long startTime = Time.monotonicNow();
+ boolean succeeded = false;
try {
ApplicationId appID = parseApplicationId(appId);
if (appID == null) {
@@ -172,7 +179,6 @@ public class TimelineCollectorWebService {
throw new NotFoundException("Application: "+ appId + " is not found");
}
- boolean isAsync = async != null && async.trim().equalsIgnoreCase("true");
if (isAsync) {
collector.putEntitiesAsync(processTimelineEntities(entities, appId,
Boolean.valueOf(isSubAppEntities)), callerUgi);
@@ -181,6 +187,7 @@ public class TimelineCollectorWebService {
Boolean.valueOf(isSubAppEntities)), callerUgi);
}
+ succeeded = true;
return Response.ok().build();
} catch (NotFoundException | ForbiddenException e) {
throw new WebApplicationException(e,
@@ -189,6 +196,13 @@ public class TimelineCollectorWebService {
LOG.error("Error putting entities", e);
throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ if (isAsync) {
+ METRICS.addAsyncPutEntitiesLatency(latency, succeeded);
+ } else {
+ METRICS.addPutEntitiesLatency(latency, succeeded);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e2e493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/PerNodeAggTimelineCollectorMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/PerNodeAggTimelineCollectorMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/PerNodeAggTimelineCollectorMetrics.java
new file mode 100644
index 0000000..0da9258
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/PerNodeAggTimelineCollectorMetrics.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.metrics;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+/**
+ * Metrics class for TimelineCollectorWebService
+ * running on each NM.
+ */
+@Metrics(about = "Aggregated metrics of TimelineCollector's running on each NM",
+ context = "timelineservice")
+final public class PerNodeAggTimelineCollectorMetrics {
+
+ private static final MetricsInfo METRICS_INFO =
+ info("PerNodeAggTimelineCollectorMetrics",
+ "Aggregated Metrics for TimelineCollector's running on each NM");
+ private static AtomicBoolean isInitialized = new AtomicBoolean(false);
+ private static PerNodeAggTimelineCollectorMetrics
+ instance = null;
+
+ @Metric(about = "PUT entities failure latency", valueName = "latency")
+ private MutableQuantiles putEntitiesFailureLatency;
+ @Metric(about = "PUT entities success latency", valueName = "latency")
+ private MutableQuantiles putEntitiesSuccessLatency;
+
+ @Metric(about = "async PUT entities failure latency", valueName = "latency")
+ private MutableQuantiles asyncPutEntitiesFailureLatency;
+ @Metric(about = "async PUT entities success latency", valueName = "latency")
+ private MutableQuantiles asyncPutEntitiesSuccessLatency;
+
+ private PerNodeAggTimelineCollectorMetrics() {
+ }
+
+ public static PerNodeAggTimelineCollectorMetrics getInstance() {
+ if (!isInitialized.get()) {
+ synchronized (PerNodeAggTimelineCollectorMetrics.class) {
+ if (instance == null) {
+ instance =
+ DefaultMetricsSystem.initialize("TimelineService").register(
+ METRICS_INFO.name(), METRICS_INFO.description(),
+ new PerNodeAggTimelineCollectorMetrics());
+ isInitialized.set(true);
+ }
+ }
+ }
+ return instance;
+ }
+
+ public synchronized static void destroy() {
+ isInitialized.set(false);
+ instance = null;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getPutEntitiesSuccessLatency() {
+ return putEntitiesSuccessLatency;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getPutEntitiesFailureLatency() {
+ return putEntitiesFailureLatency;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getAsyncPutEntitiesSuccessLatency() {
+ return asyncPutEntitiesSuccessLatency;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getAsyncPutEntitiesFailureLatency() {
+ return asyncPutEntitiesFailureLatency;
+ }
+
+ public void addPutEntitiesLatency(
+ long durationMs, boolean succeeded) {
+ if (succeeded) {
+ putEntitiesSuccessLatency.add(durationMs);
+ } else {
+ putEntitiesFailureLatency.add(durationMs);
+ }
+ }
+
+ public void addAsyncPutEntitiesLatency(
+ long durationMs, boolean succeeded) {
+ if (succeeded) {
+ asyncPutEntitiesSuccessLatency.add(durationMs);
+ } else {
+ asyncPutEntitiesFailureLatency.add(durationMs);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e2e493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/TimelineReaderMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/TimelineReaderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/TimelineReaderMetrics.java
new file mode 100644
index 0000000..3131748
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/TimelineReaderMetrics.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.metrics;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+/**
+ * Metrics class for TimelineReader.
+ */
+@Metrics(about = "Metrics for timeline reader", context = "timelineservice")
+final public class TimelineReaderMetrics {
+
+ private final static MetricsInfo METRICS_INFO = info("TimelineReaderMetrics",
+ "Metrics for TimelineReader");
+ private static AtomicBoolean isInitialized = new AtomicBoolean(false);
+ private static TimelineReaderMetrics instance = null;
+
+ @Metric(about = "GET entities failure latency", valueName = "latency")
+ private MutableQuantiles getEntitiesFailureLatency;
+ @Metric(about = "GET entities success latency", valueName = "latency")
+ private MutableQuantiles getEntitiesSuccessLatency;
+
+ @Metric(about = "GET entity types failure latency", valueName = "latency")
+ private MutableQuantiles getEntityTypesFailureLatency;
+ @Metric(about = "GET entity types success latency", valueName = "latency")
+ private MutableQuantiles getEntityTypesSuccessLatency;
+
+ private TimelineReaderMetrics() {
+ }
+
+ public static TimelineReaderMetrics getInstance() {
+ if (!isInitialized.get()) {
+ synchronized (TimelineReaderMetrics.class) {
+ if (instance == null) {
+ instance =
+ DefaultMetricsSystem.initialize("TimelineService").register(
+ METRICS_INFO.name(), METRICS_INFO.description(),
+ new TimelineReaderMetrics());
+ isInitialized.set(true);
+ }
+ }
+ }
+ return instance;
+ }
+
+ public synchronized static void destroy() {
+ isInitialized.set(false);
+ instance = null;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getGetEntitiesSuccessLatency() {
+ return getEntitiesSuccessLatency;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getGetEntitiesFailureLatency() {
+ return getEntitiesFailureLatency;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getGetEntityTypesSuccessLatency() {
+ return getEntityTypesSuccessLatency;
+ }
+
+ @VisibleForTesting
+ public MutableQuantiles getGetEntityTypesFailureLatency() {
+ return getEntityTypesFailureLatency;
+ }
+
+ public void addGetEntitiesLatency(
+ long durationMs, boolean succeeded) {
+ if (succeeded) {
+ getEntitiesSuccessLatency.add(durationMs);
+ } else {
+ getEntitiesFailureLatency.add(durationMs);
+ }
+ }
+
+ public void addGetEntityTypesLatency(
+ long durationMs, boolean succeeded) {
+ if (succeeded) {
+ getEntityTypesSuccessLatency.add(durationMs);
+ } else {
+ getEntityTypesFailureLatency.add(durationMs);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e2e493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/package-info.java
new file mode 100644
index 0000000..4cf201c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/metrics/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.server.timelineservice contains classes to be used
+ * across timeline reader and collector.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e2e493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 3a4ea2e..db48355 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.metrics.TimelineReaderMetrics;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
@@ -77,6 +78,8 @@ public class TimelineReaderWebServices {
private static final String QUERY_STRING_SEP = "?";
private static final String RANGE_DELIMITER = "-";
private static final String DATE_PATTERN = "yyyyMMdd";
+ private static final TimelineReaderMetrics METRICS =
+ TimelineReaderMetrics.getInstance();
@VisibleForTesting
static final ThreadLocal<DateFormat> DATE_FORMAT =
@@ -323,6 +326,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -343,16 +347,19 @@ public class TimelineReaderWebServices {
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntities(entities, callerUGI, entityType);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -603,6 +610,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -620,16 +628,19 @@ public class TimelineReaderWebServices {
fields, metricsLimit, metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntities(entities, callerUGI, entityType);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -697,6 +708,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
@@ -711,18 +723,21 @@ public class TimelineReaderWebServices {
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntity(entity, callerUGI);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
- (endTime - startTime) + " ms.)");
+ (Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("Timeline entity with uid: " + uId +
"is not found");
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -889,6 +904,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
@@ -901,18 +917,21 @@ public class TimelineReaderWebServices {
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntity(entity, callerUGI);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
- (endTime - startTime) + " ms.)");
+ (Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("Timeline entity {id: " + entityId +
", type: " + entityType + " } is not found");
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -955,6 +974,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
@@ -970,17 +990,20 @@ public class TimelineReaderWebServices {
entity = timelineReaderManager.getEntity(context,
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
null, metricsToRetrieve, null, null, null, null));
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entity == null) {
LOG.info("Processed URL " + url + " but flowrun not found (Took " +
- (endTime - startTime) + " ms.)");
+ (Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("Flowrun with uid: " + uId + "is not found");
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -1069,6 +1092,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
@@ -1083,20 +1107,23 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils
.createTimelineDataToRetrieve(null, metricsToRetrieve, null, null,
null, null));
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entity == null) {
LOG.info("Processed URL " + url + " but flowrun not found (Took " +
- (endTime - startTime) + " ms.)");
+ (Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("Flow run {flow name: " +
TimelineReaderWebServicesUtils.parseStr(flowName) + ", run id: " +
TimelineReaderWebServicesUtils.parseLongStr(flowRunId) +
" } is not found");
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -1161,6 +1188,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -1179,16 +1207,19 @@ public class TimelineReaderWebServices {
null, null, null, fromId),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
null, metricsToRetrieve, fields, null, null, null));
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or fromId");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -1316,6 +1347,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -1335,16 +1367,19 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils
.createTimelineDataToRetrieve(null, metricsToRetrieve, fields,
null, null, null));
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or fromId");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -1455,6 +1490,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -1470,18 +1506,21 @@ public class TimelineReaderWebServices {
TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null, null),
entityFilters, TimelineReaderWebServicesUtils.
createTimelineDataToRetrieve(null, null, null, null, null, null));
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "limit");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
} else {
checkAccess(timelineReaderManager, callerUGI, entities,
FlowActivityEntity.USER_INFO_KEY, true);
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -1549,6 +1588,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
@@ -1564,17 +1604,20 @@ public class TimelineReaderWebServices {
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForAppEntity(entity, callerUGI);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entity == null) {
LOG.info("Processed URL " + url + " but app not found" + " (Took " +
- (endTime - startTime) + " ms.)");
+ (Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("App with uid " + uId + " not found");
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -1723,6 +1766,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
@@ -1735,17 +1779,20 @@ public class TimelineReaderWebServices {
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForAppEntity(entity, callerUGI);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entity == null) {
LOG.info("Processed URL " + url + " but app not found" + " (Took " +
- (endTime - startTime) + " ms.)");
+ (Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("App " + appId + " not found");
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entity;
}
@@ -1856,6 +1903,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -1876,16 +1924,19 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -3262,6 +3313,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<String> results = null;
@@ -3270,12 +3322,15 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineReaderContext(
clusterId, userId, flowName, flowRunId, appId,
null, null, null));
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntityTypesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return results;
}
@@ -3343,6 +3398,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -3359,16 +3415,19 @@ public class TimelineReaderWebServices {
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForSubAppEntities(entities,callerUGI);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info("Processed URL " + url +
+ " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
- LOG.info("Processed URL " + url +
- " (Took " + (endTime - startTime) + " ms.)");
return entities;
}
@@ -3414,6 +3473,7 @@ public class TimelineReaderWebServices {
LOG.info("Received URL " + url + " from user "
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
+ boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
@@ -3427,15 +3487,19 @@ public class TimelineReaderWebServices {
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForSubAppEntities(entities,callerUGI);
+ succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "");
+ } finally {
+ long latency = Time.monotonicNow() - startTime;
+ METRICS.addGetEntitiesLatency(latency, succeeded);
+ LOG.info(
+ "Processed URL " + url + " (Took " + latency + " ms.)");
}
- long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
}
- LOG.info(
- "Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)");
+
return entities;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e2e493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java
new file mode 100644
index 0000000..c9ff303
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import org.apache.hadoop.yarn.server.timelineservice.metrics.PerNodeAggTimelineCollectorMetrics;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test PerNodeAggTimelineCollectorMetrics.
+ */
+public class TestPerNodeAggTimelineCollectorMetrics {
+
+ private PerNodeAggTimelineCollectorMetrics metrics;
+
+ @Test
+ public void testTimelineCollectorMetrics() {
+ Assert.assertNotNull(metrics);
+ Assert.assertEquals(10,
+ metrics.getPutEntitiesSuccessLatency().getInterval());
+ Assert.assertEquals(10,
+ metrics.getPutEntitiesFailureLatency().getInterval());
+ Assert.assertEquals(10,
+ metrics.getAsyncPutEntitiesSuccessLatency().getInterval());
+ Assert.assertEquals(10,
+ metrics.getAsyncPutEntitiesFailureLatency().getInterval());
+ }
+
+ @Before
+ public void setup() {
+ metrics = PerNodeAggTimelineCollectorMetrics.getInstance();
+ }
+
+ @After
+ public void tearDown() {
+ PerNodeAggTimelineCollectorMetrics.destroy();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e2e493/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java
new file mode 100644
index 0000000..fa74689
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.yarn.server.timelineservice.metrics.TimelineReaderMetrics;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test TimelineReaderMetrics.
+ */
+public class TestTimelineReaderMetrics {
+
+ private TimelineReaderMetrics metrics;
+
+ @Test
+ public void testTimelineReaderMetrics() {
+ Assert.assertNotNull(metrics);
+ Assert.assertEquals(10,
+ metrics.getGetEntitiesSuccessLatency().getInterval());
+ Assert.assertEquals(10,
+ metrics.getGetEntitiesFailureLatency().getInterval());
+ Assert.assertEquals(10,
+ metrics.getGetEntityTypesSuccessLatency().getInterval());
+ Assert.assertEquals(10,
+ metrics.getGetEntityTypesFailureLatency().getInterval());
+ }
+
+ @Before
+ public void setup() {
+ metrics = TimelineReaderMetrics.getInstance();
+ }
+
+ @After
+ public void tearDown() {
+ TimelineReaderMetrics.destroy();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HADOOP-15792. typo in
AzureBlobFileSystem.getIsNamespaceEnabeld. Contributed by Abhishek Modi.
Posted by in...@apache.org.
HADOOP-15792. typo in AzureBlobFileSystem.getIsNamespaceEnabeld.
Contributed by Abhishek Modi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8b86043
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8b86043
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8b86043
Branch: refs/heads/HDFS-13532
Commit: e8b8604314a2ea894b1f24939c42e782b83335aa
Parents: 58fa96b
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 2 18:57:20 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 2 18:57:20 2018 +0100
----------------------------------------------------------------------
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 18 +--
.../ITestAzureBlobFileSystemAuthorization.java | 32 ++---
.../ITestAzureBlobFileSystemBackCompat.java | 2 +-
.../ITestAzureBlobFileSystemPermission.java | 4 +-
.../ITestAzureBlobFileSystemRandomRead.java | 2 +-
.../azurebfs/ITestAzureBlobFilesystemAcl.java | 116 +++++++++----------
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 8 +-
7 files changed, 91 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b86043/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 4b521e1..d84ea5b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -553,7 +553,7 @@ public class AzureBlobFileSystem extends FileSystem {
throws IOException {
LOG.debug(
"AzureBlobFileSystem.setOwner path: {}", path);
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
super.setOwner(path, owner, group);
return;
}
@@ -584,7 +584,7 @@ public class AzureBlobFileSystem extends FileSystem {
public void setPermission(final Path path, final FsPermission permission)
throws IOException {
LOG.debug("AzureBlobFileSystem.setPermission path: {}", path);
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
super.setPermission(path, permission);
return;
}
@@ -619,7 +619,7 @@ public class AzureBlobFileSystem extends FileSystem {
throws IOException {
LOG.debug("AzureBlobFileSystem.modifyAclEntries path: {}", path.toString());
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
"modifyAclEntries is only supported by storage accounts with the "
+ "hierarchical namespace enabled.");
@@ -653,7 +653,7 @@ public class AzureBlobFileSystem extends FileSystem {
throws IOException {
LOG.debug("AzureBlobFileSystem.removeAclEntries path: {}", path);
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
"removeAclEntries is only supported by storage accounts with the "
+ "hierarchical namespace enabled.");
@@ -683,7 +683,7 @@ public class AzureBlobFileSystem extends FileSystem {
public void removeDefaultAcl(final Path path) throws IOException {
LOG.debug("AzureBlobFileSystem.removeDefaultAcl path: {}", path);
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
"removeDefaultAcl is only supported by storage accounts with the "
+ "hierarchical namespace enabled.");
@@ -711,7 +711,7 @@ public class AzureBlobFileSystem extends FileSystem {
public void removeAcl(final Path path) throws IOException {
LOG.debug("AzureBlobFileSystem.removeAcl path: {}", path);
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
"removeAcl is only supported by storage accounts with the "
+ "hierarchical namespace enabled.");
@@ -742,7 +742,7 @@ public class AzureBlobFileSystem extends FileSystem {
throws IOException {
LOG.debug("AzureBlobFileSystem.setAcl path: {}", path);
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
"setAcl is only supported by storage accounts with the hierarchical "
+ "namespace enabled.");
@@ -773,7 +773,7 @@ public class AzureBlobFileSystem extends FileSystem {
public AclStatus getAclStatus(final Path path) throws IOException {
LOG.debug("AzureBlobFileSystem.getAclStatus path: {}", path.toString());
- if (!getIsNamespaceEnabeld()) {
+ if (!getIsNamespaceEnabled()) {
throw new UnsupportedOperationException(
"getAclStatus is only supported by storage account with the "
+ "hierarchical namespace enabled.");
@@ -1005,7 +1005,7 @@ public class AzureBlobFileSystem extends FileSystem {
}
@VisibleForTesting
- boolean getIsNamespaceEnabeld() throws AzureBlobFileSystemException {
+ boolean getIsNamespaceEnabled() throws AzureBlobFileSystemException {
return abfsStore.getIsNamespaceEnabled();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b86043/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
index 1705516..d73f534 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
@@ -203,7 +203,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetOwnerAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.setOwner(TEST_WRITE_ONLY_FILE_PATH_0, "testUser", "testGroup");
}
@@ -211,7 +211,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetOwnerUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -222,7 +222,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetPermissionAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.setPermission(TEST_WRITE_ONLY_FILE_PATH_0, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
}
@@ -230,7 +230,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetPermissionUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -241,7 +241,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testModifyAclEntriesAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
fs.modifyAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
@@ -250,7 +250,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testModifyAclEntriesUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
@@ -262,7 +262,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveAclEntriesAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
fs.removeAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
@@ -271,7 +271,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveAclEntriesUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
@@ -283,7 +283,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveDefaultAclAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.removeDefaultAcl(TEST_WRITE_ONLY_FILE_PATH_0);
}
@@ -291,7 +291,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveDefaultAclUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -302,7 +302,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveAclAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.removeAcl(TEST_WRITE_ONLY_FILE_PATH_0);
}
@@ -310,7 +310,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveAclUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -321,7 +321,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetAclAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
fs.setAcl(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
@@ -330,7 +330,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetAclUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
@@ -342,7 +342,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testGetAclStatusAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
fs.getAclStatus(TEST_WRITE_THEN_READ_ONLY_PATH);
@@ -351,7 +351,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testGetAclStatusUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabled());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b86043/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
index d8940f7..22d4990 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java
@@ -45,7 +45,7 @@ public class ITestAzureBlobFileSystemBackCompat extends
public void testBlobBackCompat() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(fs.getIsNamespaceEnabeld());
+ Assume.assumeFalse(fs.getIsNamespaceEnabled());
String storageConnectionString = getBlobConnectionString();
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b86043/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemPermission.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemPermission.java
index bbb2e24..257fb4f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemPermission.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemPermission.java
@@ -76,7 +76,7 @@ public class ITestAzureBlobFileSystemPermission extends AbstractAbfsIntegrationT
public void testFilePermission() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(fs.getIsNamespaceEnabled());
fs.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE);
path = new Path(testRoot, UUID.randomUUID().toString());
@@ -92,7 +92,7 @@ public class ITestAzureBlobFileSystemPermission extends AbstractAbfsIntegrationT
@Test
public void testFolderPermission() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(fs.getIsNamespaceEnabled());
fs.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "027");
path = new Path(testRoot, UUID.randomUUID().toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b86043/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
index 38e7133..768f4bb 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java
@@ -525,7 +525,7 @@ public class ITestAzureBlobFileSystemRandomRead extends
private void createTestFile() throws Exception {
final AzureBlobFileSystem abFs = this.getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(abFs.getIsNamespaceEnabeld());
+ Assume.assumeFalse(abFs.getIsNamespaceEnabled());
FileSystem fs = this.getWasbFileSystem();
if (fs.exists(TEST_FILE_PATH)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b86043/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
index acafe03..7377132 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFilesystemAcl.java
@@ -80,7 +80,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntries() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.mkdirs(path, FsPermission.createImmutable((short) RWX_RX));
@@ -113,7 +113,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntriesOnlyAccess() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -137,7 +137,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntriesOnlyDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -160,7 +160,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntriesMinimal() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -178,7 +178,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntriesMinimalDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -198,7 +198,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntriesCustomMask() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -217,7 +217,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntriesStickyBit() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 01750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -247,7 +247,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test(expected=FileNotFoundException.class)
public void testModifyAclEntriesPathNotFound() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
// Path has not been created.
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -261,7 +261,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test (expected=Exception.class)
public void testModifyAclEntriesDefaultOnFile() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -273,7 +273,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclEntries() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -301,7 +301,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclEntriesOnlyAccess() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -326,7 +326,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclEntriesOnlyDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -353,7 +353,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclEntriesMinimal() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RWX_RW));
@@ -376,7 +376,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclEntriesMinimalDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -404,7 +404,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclEntriesStickyBit() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 01750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -432,7 +432,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test(expected=FileNotFoundException.class)
public void testRemoveAclEntriesPathNotFound() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
// Path has not been created.
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -443,7 +443,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveDefaultAcl() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -465,7 +465,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveDefaultAclOnlyAccess() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -487,7 +487,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveDefaultAclOnlyDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -503,7 +503,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveDefaultAclMinimal() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
fs.removeDefaultAcl(path);
@@ -516,7 +516,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveDefaultAclStickyBit() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 01750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -538,7 +538,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test(expected=FileNotFoundException.class)
public void testRemoveDefaultAclPathNotFound() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
// Path has not been created.
fs.removeDefaultAcl(path);
@@ -547,7 +547,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAcl() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -569,7 +569,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclMinimalAcl() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -583,7 +583,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclStickyBit() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 01750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -603,7 +603,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclOnlyDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -622,7 +622,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test(expected=FileNotFoundException.class)
public void testRemoveAclPathNotFound() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
// Path has not been created.
fs.removeAcl(path);
@@ -631,7 +631,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAcl() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -657,7 +657,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAclOnlyAccess() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -678,7 +678,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAclOnlyDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -698,7 +698,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAclMinimal() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R_R));
@@ -722,7 +722,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAclMinimalDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -742,7 +742,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAclCustomMask() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -764,7 +764,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAclStickyBit() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 01750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -790,7 +790,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test(expected=FileNotFoundException.class)
public void testSetAclPathNotFound() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
// Path has not been created.
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -804,7 +804,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test(expected=Exception.class)
public void testSetAclDefaultOnFile() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -816,7 +816,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetPermission() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -843,7 +843,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetPermissionOnlyAccess() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
fs.create(path).close();
fs.setPermission(path, FsPermission.createImmutable((short) RW_R));
@@ -865,7 +865,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetPermissionOnlyDefault() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -889,7 +889,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testDefaultAclNewFile() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -909,7 +909,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Ignore // wait umask fix to be deployed
public void testOnlyAccessAclNewFile() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -927,7 +927,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Ignore // wait investigation in service
public void testDefaultMinimalAclNewFile() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -946,7 +946,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testDefaultAclNewDir() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -973,7 +973,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Ignore // wait umask fix to be deployed
public void testOnlyAccessAclNewDir() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -991,7 +991,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Ignore // wait investigation in service
public void testDefaultMinimalAclNewDir() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -1013,7 +1013,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testDefaultAclNewFileWithMode() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -1035,7 +1035,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testDefaultAclNewDirWithMode() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) RWX_RX_RX));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -1059,7 +1059,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testDefaultAclRenamedFile() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
Path dirPath = new Path(path, "dir");
FileSystem.mkdirs(fs, dirPath, FsPermission.createImmutable((short) RWX_RX));
@@ -1081,7 +1081,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testDefaultAclRenamedDir() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
path = new Path(testRoot, UUID.randomUUID().toString());
Path dirPath = new Path(path, "dir");
FileSystem.mkdirs(fs, dirPath, FsPermission.createImmutable((short) RWX_RX));
@@ -1102,7 +1102,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testEnsureAclOperationWorksForRoot() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- assumeTrue(fs.getIsNamespaceEnabeld());
+ assumeTrue(fs.getIsNamespaceEnabled());
Path rootPath = new Path("/");
@@ -1127,7 +1127,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetOwnerForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
@@ -1144,7 +1144,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetPermissionForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
@@ -1163,7 +1163,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testModifyAclEntriesForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
try {
@@ -1180,7 +1180,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclEntriesEntriesForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
try {
@@ -1197,7 +1197,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveDefaultAclForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
try {
@@ -1211,7 +1211,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testRemoveAclForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
try {
@@ -1225,7 +1225,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testSetAclForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
try {
@@ -1242,7 +1242,7 @@ public class ITestAzureBlobFilesystemAcl extends AbstractAbfsIntegrationTest {
@Test
public void testGetAclStatusForNonNamespaceEnabledAccount() throws Exception {
final AzureBlobFileSystem fs = this.getFileSystem();
- Assume.assumeTrue(!fs.getIsNamespaceEnabeld());
+ Assume.assumeTrue(!fs.getIsNamespaceEnabled());
final Path filePath = new Path(methodName.getMethodName());
fs.create(filePath);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b86043/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
index 33a5805..fdf9788 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java
@@ -59,7 +59,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
// crate file using abfs
AzureBlobFileSystem fs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(fs.getIsNamespaceEnabeld());
+ Assume.assumeFalse(fs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@@ -93,7 +93,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(abfs.getIsNamespaceEnabeld());
+ Assume.assumeFalse(abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@@ -132,7 +132,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(abfs.getIsNamespaceEnabeld());
+ Assume.assumeFalse(abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
@@ -166,7 +166,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
//create folders
AzureBlobFileSystem abfs = getFileSystem();
// test only valid for non-namespace enabled account
- Assume.assumeFalse(abfs.getIsNamespaceEnabeld());
+ Assume.assumeFalse(abfs.getIsNamespaceEnabled());
NativeAzureFileSystem wasb = getWasbFileSystem();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-289. Volume and Bucket name
should not contain a delimiter ('/'). Contributed by chencan.
Posted by in...@apache.org.
HDDS-289. Volume and Bucket name should not contain a delimiter ('/'). Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72891fc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72891fc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72891fc9
Branch: refs/heads/HDFS-13532
Commit: 72891fc9be142ada485f06e63cd7108c54241d4b
Parents: 3c798c1
Author: Hanisha Koneru <ha...@apache.org>
Authored: Fri Sep 28 11:24:42 2018 -0700
Committer: Hanisha Koneru <ha...@apache.org>
Committed: Fri Sep 28 11:24:42 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/ozShell/TestOzoneShell.java | 55 +++++++++++++++-----
.../web/ozShell/bucket/CreateBucketHandler.java | 13 +++--
.../web/ozShell/bucket/InfoBucketHandler.java | 14 +++--
.../web/ozShell/bucket/ListBucketHandler.java | 12 ++++-
.../ozone/web/ozShell/keys/ListKeyHandler.java | 13 +++--
.../web/ozShell/volume/CreateVolumeHandler.java | 21 +++++---
.../web/ozShell/volume/InfoVolumeHandler.java | 16 ++++--
7 files changed, 108 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72891fc9/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index fbb3eae..6e73b8c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.ozShell;
+import com.google.common.base.Strings;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -31,7 +32,7 @@ import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
-
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
import org.apache.hadoop.hdds.client.ReplicationFactor;
@@ -60,17 +61,9 @@ import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import org.apache.hadoop.test.GenericTestUtils;
-
-import com.google.common.base.Strings;
-import org.apache.commons.lang3.RandomStringUtils;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -87,6 +80,12 @@ import picocli.CommandLine.ParameterException;
import picocli.CommandLine.ParseResult;
import picocli.CommandLine.RunLast;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
/**
* This test class specified for testing Ozone shell command.
*/
@@ -211,8 +210,7 @@ public class TestOzoneShell {
testCreateVolume("/////", "Volume name is required " +
"to create a volume");
testCreateVolume("/////vol/123",
- "Bucket or Volume name has " +
- "an unsupported character : /");
+ "Invalid volume name. Delimiters (/) not allowed in volume name");
}
private void testCreateVolume(String volumeName, String errorMsg)
@@ -305,6 +303,12 @@ public class TestOzoneShell {
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
+ // test infoVolume with invalid volume name
+ args = new String[] {"volume", "info",
+ url + "/" + volumeName + "/invalid-name"};
+ executeWithError(shell, args, "Invalid volume name. " +
+ "Delimiters (/) not allowed in volume name");
+
// get info for non-exist volume
args = new String[] {"volume", "info", url + "/invalid-volume"};
executeWithError(shell, args, "VOLUME_NOT_FOUND");
@@ -563,8 +567,13 @@ public class TestOzoneShell {
// test create a bucket in a non-exist volume
args = new String[] {"bucket", "create",
url + "/invalid-volume/" + bucketName};
-
executeWithError(shell, args, "Info Volume failed, error:VOLUME_NOT_FOUND");
+
+ // test createBucket with invalid bucket name
+ args = new String[] {"bucket", "create",
+ url + "/" + vol.getName() + "/" + bucketName + "/invalid-name"};
+ executeWithError(shell, args,
+ "Invalid bucket name. Delimiters (/) not allowed in bucket name");
}
@Test
@@ -618,6 +627,12 @@ public class TestOzoneShell {
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
+ // test infoBucket with invalid bucket name
+ args = new String[] {"bucket", "info",
+ url + "/" + vol.getName() + "/" + bucketName + "/invalid-name"};
+ executeWithError(shell, args,
+ "Invalid bucket name. Delimiters (/) not allowed in bucket name");
+
// test get info from a non-exist bucket
args = new String[] {"bucket", "info",
url + "/" + vol.getName() + "/invalid-bucket" + bucketName};
@@ -688,8 +703,14 @@ public class TestOzoneShell {
assertNotNull(bucket);
}
- // test -length option
+ // test listBucket with invalid volume name
String[] args = new String[] {"bucket", "list",
+ url + "/" + vol.getName() + "/invalid-name"};
+ executeWithError(shell, args, "Invalid volume name. " +
+ "Delimiters (/) not allowed in volume name");
+
+ // test -length option
+ args = new String[] {"bucket", "list",
url + "/" + vol.getName(), "--length", "100"};
execute(shell, args);
commandOutput = out.toString();
@@ -954,8 +975,14 @@ public class TestOzoneShell {
keyOutputStream.close();
}
- // test -length option
+ // test listKey with invalid bucket name
String[] args = new String[] {"key", "list",
+ url + "/" + volumeName + "/" + bucketName + "/invalid-name"};
+ executeWithError(shell, args, "Invalid bucket name. " +
+ "Delimiters (/) not allowed in bucket name");
+
+ // test -length option
+ args = new String[] {"key", "list",
url + "/" + volumeName + "/" + bucketName, "--length", "100"};
execute(shell, args);
commandOutput = out.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72891fc9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
index bd8db60..49bbc18 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
@@ -50,9 +50,16 @@ public class CreateBucketHandler extends Handler {
URI ozoneURI = verifyURI(uri);
Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 2) {
- throw new OzoneClientException(
- "volume and bucket name required in createBucket");
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 2) {
+ String errorMessage;
+ if (pathNameCount < 2) {
+ errorMessage = "volume and bucket name required in createBucket";
+ } else {
+ errorMessage = "Invalid bucket name. Delimiters (/) not allowed in " +
+ "bucket name";
+ }
+ throw new OzoneClientException(errorMessage);
}
String volumeName = path.getName(0).toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72891fc9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
index 4122b72..18f8a0b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
@@ -50,10 +50,16 @@ public class InfoBucketHandler extends Handler {
String volumeName, bucketName;
URI ozoneURI = verifyURI(uri);
Path path = Paths.get(ozoneURI.getPath());
-
- if (path.getNameCount() < 2) {
- throw new OzoneClientException(
- "volume and bucket name required in info Bucket");
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 2) {
+ String errorMessage;
+ if (pathNameCount < 2) {
+ errorMessage = "volume and bucket name required in infoBucket";
+ } else {
+ errorMessage = "Invalid bucket name. Delimiters (/) not allowed in " +
+ "bucket name";
+ }
+ throw new OzoneClientException(errorMessage);
}
volumeName = path.getName(0).toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72891fc9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
index d8c824b..7c3029a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
@@ -71,8 +71,16 @@ public class ListBucketHandler extends Handler {
URI ozoneURI = verifyURI(uri);
Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 1) {
- throw new OzoneClientException("volume is required in listBucket");
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 1) {
+ String errorMessage;
+ if (pathNameCount < 1) {
+ errorMessage = "volume is required in listBucket";
+ } else {
+ errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
+ "volume name";
+ }
+ throw new OzoneClientException(errorMessage);
}
if (maxBuckets < 1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72891fc9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
index 3c019a6..c539d60 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
@@ -71,9 +71,16 @@ public class ListKeyHandler extends Handler {
URI ozoneURI = verifyURI(uri);
Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 2) {
- throw new OzoneClientException(
- "volume/bucket is required in listKey");
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 2) {
+ String errorMessage;
+ if (pathNameCount < 2) {
+ errorMessage = "volume/bucket is required in listKey";
+ } else {
+ errorMessage = "Invalid bucket name. Delimiters (/) not allowed in " +
+ "bucket name";
+ }
+ throw new OzoneClientException(errorMessage);
}
if (maxKeys < 1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72891fc9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
index 2697639..de0fbaa 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
import java.net.URI;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
@@ -64,15 +66,20 @@ public class CreateVolumeHandler extends Handler {
public Void call() throws Exception {
URI ozoneURI = verifyURI(uri);
-
- // we need to skip the slash in the URI path
- // getPath returns /volumeName needs to remove the initial slash.
- String volumeName = ozoneURI.getPath().replaceAll("^/+", "");
- if (volumeName.isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to create a volume");
+ Path path = Paths.get(ozoneURI.getPath());
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 1) {
+ String errorMessage;
+ if (pathNameCount < 1) {
+ errorMessage = "Volume name is required to create a volume";
+ } else {
+ errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
+ "volume name";
+ }
+ throw new OzoneClientException(errorMessage);
}
+ String volumeName = ozoneURI.getPath().replaceAll("^/+", "");
if (isVerbose()) {
System.out.printf("Volume name : %s%n", volumeName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72891fc9/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
index 60a8c6d..16c0342 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
import java.net.URI;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
@@ -47,9 +49,17 @@ public class InfoVolumeHandler extends Handler{
public Void call() throws Exception {
URI ozoneURI = verifyURI(uri);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to get info of a volume");
+ Path path = Paths.get(ozoneURI.getPath());
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 1) {
+ String errorMessage;
+ if (pathNameCount < 1) {
+ errorMessage = "Volume name is required to get info of a volume";
+ } else {
+ errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
+ "volume name";
+ }
+ throw new OzoneClientException(errorMessage);
}
// we need to skip the slash in the URI path
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-8800. Updated documentation
of Submarine with latest examples. Contributed by Wangda Tan.
Posted by in...@apache.org.
YARN-8800. Updated documentation of Submarine with latest examples. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19ad5be6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19ad5be6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19ad5be6
Branch: refs/heads/HDFS-13532
Commit: 19ad5be6517765b31b4afaccbaadc8268627c568
Parents: 72891fc
Author: Sunil G <su...@apache.org>
Authored: Sat Sep 29 00:01:04 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Sat Sep 29 00:01:04 2018 +0530
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 6 +
.../hadoop-yarn-submarine/README.md | 6 +-
.../base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 | 69 +++
.../base/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 | 67 +++
.../src/main/docker/build-all.sh | 32 ++
.../ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 | 22 +
.../ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 | 22 +
.../cifar10_estimator_tf_1.8.0/README.md | 542 ++++++++++++++++++
.../cifar10_estimator_tf_1.8.0/cifar10.py | 113 ++++
.../cifar10_estimator_tf_1.8.0/cifar10_main.py | 521 +++++++++++++++++
.../cifar10_estimator_tf_1.8.0/cifar10_model.py | 80 +++
.../cifar10_estimator_tf_1.8.0/cifar10_utils.py | 154 +++++
.../generate_cifar10_tfrecords.py | 114 ++++
.../cifar10_estimator_tf_1.8.0/model_base.py | 219 +++++++
.../zeppelin-notebook-example/Dockerfile.gpu | 75 +++
.../zeppelin-notebook-example/run_container.sh | 22 +
.../docker/zeppelin-notebook-example/shiro.ini | 120 ++++
.../zeppelin-notebook-example/zeppelin-site.xml | 569 +++++++++++++++++++
.../src/site/DeveloperGuide.md | 26 -
.../src/site/QuickStart.md | 134 -----
.../src/site/markdown/DeveloperGuide.md | 24 +
.../src/site/markdown/Examples.md | 21 +
.../src/site/markdown/Index.md | 42 ++
.../src/site/markdown/QuickStart.md | 174 ++++++
.../markdown/RunningDistributedCifar10TFJobs.md | 162 ++++++
.../src/site/markdown/RunningZeppelinOnYARN.md | 37 ++
.../src/site/markdown/WriteDockerfile.md | 117 ++++
.../src/site/resources/css/site.css | 29 +
.../src/site/resources/images/job-logs-ui.png | Bin 0 -> 229944 bytes
.../images/multiple-tensorboard-jobs.png | Bin 0 -> 184717 bytes
.../resources/images/tensorboard-service.png | Bin 0 -> 107567 bytes
.../hadoop-yarn-submarine/src/site/site.xml | 28 +
32 files changed, 3385 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index b40dbfc..2b6058e 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -180,6 +180,12 @@
<item name="System Services" href="hadoop-yarn/hadoop-yarn-site/yarn-service/SystemServices.html"/>
</menu>
+ <menu name="Submarine" inherit="top">
+ <item name="Index" href="hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/Index.html"/>
+ <item name="QuickStart" href="hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/QuickStart.html"/>
+ <item name="Examples" href="hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/Examples.html"/>
+ </menu>
+
<menu name="Hadoop Compatible File Systems" inherit="top">
<item name="Aliyun OSS" href="hadoop-aliyun/tools/hadoop-aliyun/index.html"/>
<item name="Amazon S3" href="hadoop-aws/tools/hadoop-aws/index.html"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/README.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/README.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/README.md
index 3e04730..cb2e2da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/README.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/README.md
@@ -48,6 +48,8 @@ Goals of Submarine:
- Support launch tensorboard for training jobs if user specified.
- Support customized DNS name for roles (like tensorboard.$user.$domain:6006)
-Please jump to [QuickStart](src/site/QuickStart.md) guide to quickly understand how to use this framework.
+Please jump to [QuickStart](src/site/markdown/QuickStart.md) guide to quickly understand how to use this framework.
-If you're a developer, please find [Developer](src/site/DeveloperGuide.md) guide for more details.
+Please jump to [Examples](src/site/markdown/Examples.md) to try other examples like running Distributed Tensorflow Training for CIFAR 10.
+
+If you're a developer, please find [Developer](src/site/markdown/DeveloperGuide.md) guide for more details.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0
new file mode 100644
index 0000000..f2446a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:16.04
+
+LABEL maintainer="Craig Citro <cr...@google.com>"
+
+# Pick up some TF dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ build-essential \
+ curl \
+ libfreetype6-dev \
+ libpng12-dev \
+ libzmq3-dev \
+ pkg-config \
+ python \
+ python-dev \
+ rsync \
+ software-properties-common \
+ unzip \
+ && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py && \
+ rm get-pip.py
+
+RUN pip --no-cache-dir install \
+ Pillow \
+ h5py \
+ ipykernel \
+ jupyter \
+ matplotlib \
+ numpy \
+ pandas \
+ scipy \
+ sklearn \
+ && \
+ python -m ipykernel.kernelspec
+
+# --- DO NOT EDIT OR DELETE BETWEEN THE LINES --- #
+# These lines will be edited automatically by parameterized_docker_build.sh. #
+# COPY _PIP_FILE_ /
+# RUN pip --no-cache-dir install /_PIP_FILE_
+# RUN rm -f /_PIP_FILE_
+
+# Install TensorFlow CPU version from central repo
+RUN pip --no-cache-dir install \
+ http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.8.0-cp27-none-linux_x86_64.whl
+
+RUN apt-get update && apt-get install git -y
+
+RUN apt-get update && apt-get install -y openjdk-8-jdk wget
+RUN wget http://apache.cs.utah.edu/hadoop/common/hadoop-3.1.1/hadoop-3.1.1.tar.gz
+RUN tar zxf hadoop-3.1.1.tar.gz
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0
new file mode 100644
index 0000000..dee6e19
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/base/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+
+# Pick up some TF dependencies
+RUN apt-get update && apt-get install -y --allow-downgrades --no-install-recommends \
+ build-essential \
+ cuda-command-line-tools-9-0 \
+ cuda-cublas-9-0 \
+ cuda-cufft-9-0 \
+ cuda-curand-9-0 \
+ cuda-cusolver-9-0 \
+ cuda-cusparse-9-0 \
+ curl \
+ libcudnn7=7.0.5.15-1+cuda9.0 \
+ libfreetype6-dev \
+ libpng12-dev \
+ libzmq3-dev \
+ pkg-config \
+ python \
+ python-dev \
+ rsync \
+ software-properties-common \
+ unzip \
+ && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py && \
+ rm get-pip.py
+
+RUN pip --no-cache-dir install \
+ Pillow \
+ h5py \
+ ipykernel \
+ jupyter \
+ matplotlib \
+ numpy \
+ pandas \
+ scipy \
+ sklearn \
+ && \
+ python -m ipykernel.kernelspec
+
+# Install TensorFlow GPU version.
+RUN pip --no-cache-dir install \
+ http://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.8.0-cp27-none-linux_x86_64.whl
+RUN apt-get update && apt-get install git -y
+
+RUN apt-get update && apt-get install -y openjdk-8-jdk wget
+RUN wget http://apache.cs.utah.edu/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.tar.gz
+RUN tar zxf hadoop-3.1.0.tar.gz
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/build-all.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/build-all.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/build-all.sh
new file mode 100755
index 0000000..ad3a935
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/build-all.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "Building base images"
+
+set -e
+
+cd base/ubuntu-16.04
+
+docker build . -f Dockerfile.cpu.tf_1.8.0 -t tf-1.8.0-cpu-base:0.0.1
+docker build . -f Dockerfile.gpu.tf_1.8.0 -t tf-1.8.0-gpu-base:0.0.1
+
+echo "Finished building base images"
+
+cd ../../with-cifar10-models/ubuntu-16.04
+
+docker build . -f Dockerfile.cpu.tf_1.8.0 -t tf-1.8.0-cpu:0.0.1
+docker build . -f Dockerfile.gpu.tf_1.8.0 -t tf-1.8.0-gpu:0.0.1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0
new file mode 100644
index 0000000..1087d61
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM tf-1.8.0-cpu-base:0.0.1
+
+# Include models
+RUN mkdir /test
+ADD cifar10_estimator_tf_1.8.0 /test/cifar10_estimator
+RUN chown -R nobody /test
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0
new file mode 100644
index 0000000..d1f829f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM tf-1.8.0-gpu-base:0.0.1
+
+# Include models
+RUN mkdir /test
+ADD cifar10_estimator_tf_1.8.0 /test/cifar10_estimator
+RUN chown -R nobody /test
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/README.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/README.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/README.md
new file mode 100644
index 0000000..5b4ae34
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/README.md
@@ -0,0 +1,542 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+(Copied from https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator)
+
+CIFAR-10 is a common benchmark in machine learning for image recognition.
+
+http://www.cs.toronto.edu/~kriz/cifar.html
+
+Code in this directory focuses on how to use TensorFlow Estimators to train and
+evaluate a CIFAR-10 ResNet model on:
+
+* A single host with one CPU;
+* A single host with multiple GPUs;
+* Multiple hosts with CPU or multiple GPUs;
+
+Before trying to run the model we highly encourage you to read all the README.
+
+## Prerequisite
+
+1. [Install](https://www.tensorflow.org/install/) TensorFlow version 1.2.1 or
+later.
+
+2. Download the CIFAR-10 dataset and generate TFRecord files using the provided
+script. The script and associated command below will download the CIFAR-10
+dataset and then generate a TFRecord for the training, validation, and
+evaluation datasets.
+
+```shell
+python generate_cifar10_tfrecords.py --data-dir=${PWD}/cifar-10-data
+```
+
+After running the command above, you should see the following files in the
+--data-dir (```ls -R cifar-10-data```):
+
+* train.tfrecords
+* validation.tfrecords
+* eval.tfrecords
+
+
+## Training on a single machine with GPUs or CPU
+
+Run the training on CPU only. After training, it runs the evaluation.
+
+```
+python cifar10_main.py --data-dir=${PWD}/cifar-10-data \
+ --job-dir=/tmp/cifar10 \
+ --num-gpus=0 \
+ --train-steps=1000
+```
+
+Run the model on 2 GPUs using CPU as parameter server. After training, it runs
+the evaluation.
+```
+python cifar10_main.py --data-dir=${PWD}/cifar-10-data \
+ --job-dir=/tmp/cifar10 \
+ --num-gpus=2 \
+ --train-steps=1000
+```
+
+Run the model on 2 GPUs using GPU as parameter server.
+It will run an experiment, which for local setting basically means it will run
+stop training
+a couple of times to perform evaluation.
+
+```
+python cifar10_main.py --data-dir=${PWD}/cifar-10-data \
+ --job-dir=/tmp/cifar10 \
+ --variable-strategy GPU \
+ --num-gpus=2 \
+```
+
+There are more command line flags to play with; run
+`python cifar10_main.py --help` for details.
+
+## Run distributed training
+
+### (Optional) Running on Google Cloud Machine Learning Engine
+
+This example can be run on Google Cloud Machine Learning Engine (ML Engine),
+which will configure the environment and take care of running workers,
+parameters servers, and masters in a fault tolerant way.
+
+To install the command line tool, and set up a project and billing, see the
+quickstart [here](https://cloud.google.com/ml-engine/docs/quickstarts/command-line).
+
+You'll also need a Google Cloud Storage bucket for the data. If you followed the
+instructions above, you can just run:
+
+```
+MY_BUCKET=gs://<my-bucket-name>
+gsutil cp -r ${PWD}/cifar-10-data $MY_BUCKET/
+```
+
+Then run the following command from the `tutorials/image` directory of this
+repository (the parent directory of this README):
+
+```
+gcloud ml-engine jobs submit training cifarmultigpu \
+ --runtime-version 1.2 \
+ --job-dir=$MY_BUCKET/model_dirs/cifarmultigpu \
+ --config cifar10_estimator/cmle_config.yaml \
+ --package-path cifar10_estimator/ \
+ --module-name cifar10_estimator.cifar10_main \
+ -- \
+ --data-dir=$MY_BUCKET/cifar-10-data \
+ --num-gpus=4 \
+ --train-steps=1000
+```
+
+
+### Set TF_CONFIG
+
+Considering that you already have multiple hosts configured, all you need is a
+`TF_CONFIG` environment variable on each host. You can set up the hosts manually
+or check [tensorflow/ecosystem](https://github.com/tensorflow/ecosystem) for
+instructions about how to set up a Cluster.
+
+The `TF_CONFIG` will be used by the `RunConfig` to know the existing hosts and
+their task: `master`, `ps` or `worker`.
+
+Here's an example of `TF_CONFIG`.
+
+```python
+cluster = {'master': ['master-ip:8000'],
+ 'ps': ['ps-ip:8000'],
+ 'worker': ['worker-ip:8000']}
+
+TF_CONFIG = json.dumps(
+ {'cluster': cluster,
+ 'task': {'type': master, 'index': 0},
+ 'model_dir': 'gs://<bucket_path>/<dir_path>',
+ 'environment': 'cloud'
+ })
+```
+
+*Cluster*
+
+A cluster spec, which is basically a dictionary that describes all of the tasks
+in the cluster. More about it [here](https://www.tensorflow.org/deploy/distributed).
+
+In this cluster spec we are defining a cluster with 1 master, 1 ps and 1 worker.
+
+* `ps`: saves the parameters among all workers. All workers can
+ read/write/update the parameters for model via ps. As some models are
+ extremely large the parameters are shared among the ps (each ps stores a
+ subset).
+
+* `worker`: does the training.
+
+* `master`: basically a special worker, it does training, but also restores and
+ saves checkpoints and do evaluation.
+
+*Task*
+
+The Task defines what is the role of the current node, for this example the node
+is the master on index 0 on the cluster spec, the task will be different for
+each node. An example of the `TF_CONFIG` for a worker would be:
+
+```python
+cluster = {'master': ['master-ip:8000'],
+ 'ps': ['ps-ip:8000'],
+ 'worker': ['worker-ip:8000']}
+
+TF_CONFIG = json.dumps(
+ {'cluster': cluster,
+ 'task': {'type': worker, 'index': 0},
+ 'model_dir': 'gs://<bucket_path>/<dir_path>',
+ 'environment': 'cloud'
+ })
+```
+
+*Model_dir*
+
+This is the path where the master will save the checkpoints, graph and
+TensorBoard files. For a multi host environment you may want to use a
+Distributed File System, Google Storage and DFS are supported.
+
+*Environment*
+
+By the default environment is *local*, for a distributed setting we need to
+change it to *cloud*.
+
+### Running script
+
+Once you have a `TF_CONFIG` configured properly on each host you're ready to run
+on distributed settings.
+
+#### Master
+Run this on master:
+Runs an Experiment in sync mode on 4 GPUs using CPU as parameter server for
+40000 steps. It will run evaluation a couple of times during training. The
+num_workers arugument is used only to update the learning rate correctly. Make
+sure the model_dir is the same as defined on the TF_CONFIG.
+
+```shell
+python cifar10_main.py --data-dir=gs://path/cifar-10-data \
+ --job-dir=gs://path/model_dir/ \
+ --num-gpus=4 \
+ --train-steps=40000 \
+ --sync \
+ --num-workers=2
+```
+
+*Output:*
+
+```shell
+INFO:tensorflow:Using model_dir in TF_CONFIG: gs://path/model_dir/
+INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_num_ps_replicas': 1, '_keep_checkpoint_max': 5, '_task_type': u'master', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fd16fb2be10>, '_model_dir': 'gs://path/model_dir/', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_session_config': intra_op_parallelism_threads: 1
+gpu_options {
+}
+allow_soft_placement: true
+, '_tf_random_seed': None, '_environment': u'cloud', '_num_worker_replicas': 1, '_task_id': 0, '_save_summary_steps': 100, '_tf_config': gpu_options {
+ per_process_gpu_memory_fraction: 1.0
+}
+, '_evaluation_master': '', '_master': u'grpc://master-ip:8000'}
+...
+2017-08-01 19:59:26.496208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties:
+name: Tesla K80
+major: 3 minor: 7 memoryClockRate (GHz) 0.8235
+pciBusID 0000:00:04.0
+Total memory: 11.17GiB
+Free memory: 11.09GiB
+2017-08-01 19:59:26.775660: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 1 with properties:
+name: Tesla K80
+major: 3 minor: 7 memoryClockRate (GHz) 0.8235
+pciBusID 0000:00:05.0
+Total memory: 11.17GiB
+Free memory: 11.10GiB
+...
+2017-08-01 19:59:29.675171: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: (?, 16, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: (?, 32, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: (?, 64)
+INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: (?, 11)
+INFO:tensorflow:SyncReplicasV2: replicas_to_aggregate=1; total_num_replicas=1
+INFO:tensorflow:Create CheckpointSaverHook.
+INFO:tensorflow:Restoring parameters from gs://path/model_dir/model.ckpt-0
+2017-08-01 19:59:37.560775: I tensorflow/core/distributed_runtime/master_session.cc:999] Start master session 156fcb55fe6648d6 with config:
+intra_op_parallelism_threads: 1
+gpu_options {
+ per_process_gpu_memory_fraction: 1
+}
+allow_soft_placement: true
+
+INFO:tensorflow:Saving checkpoints for 1 into gs://path/model_dir/model.ckpt.
+INFO:tensorflow:loss = 1.20682, step = 1
+INFO:tensorflow:loss = 1.20682, learning_rate = 0.1
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: (?, 16, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: (?, 32, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: (?, 64)
+INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: (?, 11)
+INFO:tensorflow:SyncReplicasV2: replicas_to_aggregate=2; total_num_replicas=2
+INFO:tensorflow:Starting evaluation at 2017-08-01-20:00:14
+2017-08-01 20:00:15.745881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:0) -> (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0)
+2017-08-01 20:00:15.745949: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:1) -> (device: 1, name: Tesla K80, pci bus id: 0000:00:05.0)
+2017-08-01 20:00:15.745958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:2) -> (device: 2, name: Tesla K80, pci bus id: 0000:00:06.0)
+2017-08-01 20:00:15.745964: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:3) -> (device: 3, name: Tesla K80, pci bus id: 0000:00:07.0)
+2017-08-01 20:00:15.745969: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:4) -> (device: 4, name: Tesla K80, pci bus id: 0000:00:08.0)
+2017-08-01 20:00:15.745975: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:5) -> (device: 5, name: Tesla K80, pci bus id: 0000:00:09.0)
+2017-08-01 20:00:15.745987: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:6) -> (device: 6, name: Tesla K80, pci bus id: 0000:00:0a.0)
+2017-08-01 20:00:15.745997: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1030] Creating TensorFlow device (/gpu:7) -> (device: 7, name: Tesla K80, pci bus id: 0000:00:0b.0)
+INFO:tensorflow:Restoring parameters from gs://path/model_dir/model.ckpt-10023
+INFO:tensorflow:Evaluation [1/100]
+INFO:tensorflow:Evaluation [2/100]
+INFO:tensorflow:Evaluation [3/100]
+INFO:tensorflow:Evaluation [4/100]
+INFO:tensorflow:Evaluation [5/100]
+INFO:tensorflow:Evaluation [6/100]
+INFO:tensorflow:Evaluation [7/100]
+INFO:tensorflow:Evaluation [8/100]
+INFO:tensorflow:Evaluation [9/100]
+INFO:tensorflow:Evaluation [10/100]
+INFO:tensorflow:Evaluation [11/100]
+INFO:tensorflow:Evaluation [12/100]
+INFO:tensorflow:Evaluation [13/100]
+...
+INFO:tensorflow:Evaluation [100/100]
+INFO:tensorflow:Finished evaluation at 2017-08-01-20:00:31
+INFO:tensorflow:Saving dict for global step 1: accuracy = 0.0994, global_step = 1, loss = 630.425
+```
+
+#### Worker
+
+Run this on worker:
+Runs an Experiment in sync mode on 4 GPUs using CPU as parameter server for
+40000 steps. It will run evaluation a couple of times during training. Make sure
+the model_dir is the same as defined on the TF_CONFIG.
+
+```shell
+python cifar10_main.py --data-dir=gs://path/cifar-10-data \
+ --job-dir=gs://path/model_dir/ \
+ --num-gpus=4 \
+ --train-steps=40000 \
+ --sync
+```
+
+*Output:*
+
+```shell
+INFO:tensorflow:Using model_dir in TF_CONFIG: gs://path/model_dir/
+INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600,
+'_num_ps_replicas': 1, '_keep_checkpoint_max': 5, '_task_type': u'worker',
+'_is_chief': False, '_cluster_spec':
+<tensorflow.python.training.server_lib.ClusterSpec object at 0x7f6918438e10>,
+'_model_dir': 'gs://<path>/model_dir/',
+'_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000,
+'_session_config': intra_op_parallelism_threads: 1
+gpu_options {
+}
+allow_soft_placement: true
+, '_tf_random_seed': None, '_environment': u'cloud', '_num_worker_replicas': 1,
+'_task_id': 0, '_save_summary_steps': 100, '_tf_config': gpu_options {
+ per_process_gpu_memory_fraction: 1.0
+ }
+...
+2017-08-01 19:59:26.496208: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 0 with properties:
+name: Tesla K80
+major: 3 minor: 7 memoryClockRate (GHz) 0.8235
+pciBusID 0000:00:04.0
+Total memory: 11.17GiB
+Free memory: 11.09GiB
+2017-08-01 19:59:26.775660: I tensorflow/core/common_runtime/gpu/gpu_device.cc:940] Found device 1 with properties:
+name: Tesla K80
+major: 3 minor: 7 memoryClockRate (GHz) 0.8235
+pciBusID 0000:00:05.0
+Total memory: 11.17GiB
+Free memory: 11.10GiB
+...
+2017-08-01 19:59:29.675171: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_1/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_2/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_3/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_4/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_5/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage/residual_v1_6/: (?, 16, 32, 32)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/avg_pool/: (?, 16, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_1/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_2/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_3/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_4/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_5/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_1/residual_v1_6/: (?, 32, 16, 16)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/avg_pool/: (?, 32, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_1/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_2/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_3/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_4/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_5/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/stage_2/residual_v1_6/: (?, 64, 8, 8)
+INFO:tensorflow:image after unit resnet/tower_0/global_avg_pool/: (?, 64)
+INFO:tensorflow:image after unit resnet/tower_0/fully_connected/: (?, 11)
+INFO:tensorflow:SyncReplicasV2: replicas_to_aggregate=2; total_num_replicas=2
+INFO:tensorflow:Create CheckpointSaverHook.
+2017-07-31 22:38:04.629150: I
+tensorflow/core/distributed_runtime/master.cc:209] CreateSession still waiting
+for response from worker: /job:master/replica:0/task:0
+2017-07-31 22:38:09.263492: I
+tensorflow/core/distributed_runtime/master_session.cc:999] Start master
+session cc58f93b1e259b0c with config:
+intra_op_parallelism_threads: 1
+gpu_options {
+per_process_gpu_memory_fraction: 1
+}
+allow_soft_placement: true
+INFO:tensorflow:loss = 5.82382, step = 0
+INFO:tensorflow:loss = 5.82382, learning_rate = 0.8
+INFO:tensorflow:Average examples/sec: 1116.92 (1116.92), step = 10
+INFO:tensorflow:Average examples/sec: 1233.73 (1377.83), step = 20
+INFO:tensorflow:Average examples/sec: 1485.43 (2509.3), step = 30
+INFO:tensorflow:Average examples/sec: 1680.27 (2770.39), step = 40
+INFO:tensorflow:Average examples/sec: 1825.38 (2788.78), step = 50
+INFO:tensorflow:Average examples/sec: 1929.32 (2697.27), step = 60
+INFO:tensorflow:Average examples/sec: 2015.17 (2749.05), step = 70
+INFO:tensorflow:loss = 37.6272, step = 79 (19.554 sec)
+INFO:tensorflow:loss = 37.6272, learning_rate = 0.8 (19.554 sec)
+INFO:tensorflow:Average examples/sec: 2074.92 (2618.36), step = 80
+INFO:tensorflow:Average examples/sec: 2132.71 (2744.13), step = 90
+INFO:tensorflow:Average examples/sec: 2183.38 (2777.21), step = 100
+INFO:tensorflow:Average examples/sec: 2224.4 (2739.03), step = 110
+INFO:tensorflow:Average examples/sec: 2240.28 (2431.26), step = 120
+INFO:tensorflow:Average examples/sec: 2272.12 (2739.32), step = 130
+INFO:tensorflow:Average examples/sec: 2300.68 (2750.03), step = 140
+INFO:tensorflow:Average examples/sec: 2325.81 (2745.63), step = 150
+INFO:tensorflow:Average examples/sec: 2347.14 (2721.53), step = 160
+INFO:tensorflow:Average examples/sec: 2367.74 (2754.54), step = 170
+INFO:tensorflow:loss = 27.8453, step = 179 (18.893 sec)
+...
+```
+
+#### PS
+
+Run this on ps:
+The ps will not do training so most of the arguments won't affect the execution
+
+```shell
+python cifar10_main.py --job-dir=gs://path/model_dir/
+```
+
+*Output:*
+
+```shell
+INFO:tensorflow:Using model_dir in TF_CONFIG: gs://path/model_dir/
+INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_num_ps_replicas': 1, '_keep_checkpoint_max': 5, '_task_type': u'ps', '_is_chief': False, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f48f1addf90>, '_model_dir': 'gs://path/model_dir/', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_session_config': intra_op_parallelism_threads: 1
+gpu_options {
+}
+allow_soft_placement: true
+, '_tf_random_seed': None, '_environment': u'cloud', '_num_worker_replicas': 1, '_task_id': 0, '_save_summary_steps': 100, '_tf_config': gpu_options {
+ per_process_gpu_memory_fraction: 1.0
+}
+, '_evaluation_master': '', '_master': u'grpc://master-ip:8000'}
+2017-07-31 22:54:58.928088: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job master -> {0 -> master-ip:8000}
+2017-07-31 22:54:58.928153: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:8000}
+2017-07-31 22:54:58.928160: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> worker-ip:8000}
+2017-07-31 22:54:58.929873: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000
+```
+
+## Visualizing results with TensorBoard
+
+When using Estimators you can also visualize your data in TensorBoard, with no
+changes in your code. You can use TensorBoard to visualize your TensorFlow
+graph, plot quantitative metrics about the execution of your graph, and show
+additional data like images that pass through it.
+
+You'll see something similar to this if you "point" TensorBoard to the
+`job dir` parameter you used to train or evaluate your model.
+
+Check TensorBoard during training or after it. Just point TensorBoard to the
+model_dir you chose on the previous step.
+
+```shell
+tensorboard --log-dir="<job dir>"
+```
+
+## Warnings
+
+When runninng `cifar10_main.py` with `--sync` argument you may see an error
+similar to:
+
+```python
+File "cifar10_main.py", line 538, in <module>
+ tf.app.run()
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 48, in run
+ _sys.exit(main(_sys.argv[:1] + flags_passthrough))
+File "cifar10_main.py", line 518, in main
+ hooks), run_config=config)
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 210, in run
+ return _execute_schedule(experiment, schedule)
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/learn_runner.py", line 47, in _execute_schedule
+ return task()
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 501, in train_and_evaluate
+ hooks=self._eval_hooks)
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/experiment.py", line 681, in _call_evaluate
+ hooks=hooks)
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py", line 292, in evaluate
+ name=name)
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py", line 638, in _evaluate_model
+ features, labels, model_fn_lib.ModeKeys.EVAL)
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/estimator/estimator.py", line 545, in _call_model_fn
+ features=features, labels=labels, **kwargs)
+File "cifar10_main.py", line 331, in _resnet_model_fn
+ gradvars, global_step=tf.train.get_global_step())
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/sync_replicas_optimizer.py", line 252, in apply_gradients
+ variables.global_variables())
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py", line 170, in wrapped
+ return _add_should_use_warning(fn(*args, **kwargs))
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py", line 139, in _add_should_use_warning
+ wrapped = TFShouldUseWarningWrapper(x)
+File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/util/tf_should_use.py", line 96, in __init__
+ stack = [s.strip() for s in traceback.format_stack()]
+```
+
+This should not affect your training, and should be fixed on the next releases.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10.py
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10.py
new file mode 100644
index 0000000..6903e8d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10.py
@@ -0,0 +1,113 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""CIFAR-10 data set.
+
+See http://www.cs.toronto.edu/~kriz/cifar.html.
+"""
+import os
+
+import tensorflow as tf
+
+HEIGHT = 32
+WIDTH = 32
+DEPTH = 3
+
+
+class Cifar10DataSet(object):
+ """Cifar10 data set.
+
+ Described by http://www.cs.toronto.edu/~kriz/cifar.html.
+ """
+
+ def __init__(self, data_dir, subset='train', use_distortion=True):
+ self.data_dir = data_dir
+ self.subset = subset
+ self.use_distortion = use_distortion
+
+ def get_filenames(self):
+ if self.subset in ['train', 'validation', 'eval']:
+ return [os.path.join(self.data_dir, self.subset + '.tfrecords')]
+ else:
+ raise ValueError('Invalid data subset "%s"' % self.subset)
+
+ def parser(self, serialized_example):
+ """Parses a single tf.Example into image and label tensors."""
+ # Dimensions of the images in the CIFAR-10 dataset.
+ # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
+ # input format.
+ features = tf.parse_single_example(
+ serialized_example,
+ features={
+ 'image': tf.FixedLenFeature([], tf.string),
+ 'label': tf.FixedLenFeature([], tf.int64),
+ })
+ image = tf.decode_raw(features['image'], tf.uint8)
+ image.set_shape([DEPTH * HEIGHT * WIDTH])
+
+ # Reshape from [depth * height * width] to [depth, height, width].
+ image = tf.cast(
+ tf.transpose(tf.reshape(image, [DEPTH, HEIGHT, WIDTH]), [1, 2, 0]),
+ tf.float32)
+ label = tf.cast(features['label'], tf.int32)
+
+ # Custom preprocessing.
+ image = self.preprocess(image)
+
+ return image, label
+
+ def make_batch(self, batch_size):
+ """Read the images and labels from 'filenames'."""
+ filenames = self.get_filenames()
+ # Repeat infinitely.
+ dataset = tf.data.TFRecordDataset(filenames).repeat()
+
+ # Parse records.
+ dataset = dataset.map(
+ self.parser)
+
+ # Potentially shuffle records.
+ if self.subset == 'train':
+ min_queue_examples = int(
+ Cifar10DataSet.num_examples_per_epoch(self.subset) * 0.4)
+ # Ensure that the capacity is sufficiently large to provide good random
+ # shuffling.
+ dataset = dataset.shuffle(buffer_size=min_queue_examples + 3 * batch_size)
+
+ # Batch it up.
+ dataset = dataset.batch(batch_size)
+ iterator = dataset.make_one_shot_iterator()
+ image_batch, label_batch = iterator.get_next()
+
+ return image_batch, label_batch
+
+ def preprocess(self, image):
+ """Preprocess a single image in [height, width, depth] layout."""
+ if self.subset == 'train' and self.use_distortion:
+ # Pad 4 pixels on each dimension of feature map, done in mini-batch
+ image = tf.image.resize_image_with_crop_or_pad(image, 40, 40)
+ image = tf.random_crop(image, [HEIGHT, WIDTH, DEPTH])
+ image = tf.image.random_flip_left_right(image)
+ return image
+
+ @staticmethod
+ def num_examples_per_epoch(subset='train'):
+ if subset == 'train':
+ return 45000
+ elif subset == 'validation':
+ return 5000
+ elif subset == 'eval':
+ return 10000
+ else:
+ raise ValueError('Invalid data subset "%s"' % subset)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_main.py
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_main.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_main.py
new file mode 100644
index 0000000..086c95b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_main.py
@@ -0,0 +1,521 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ResNet model for classifying images from CIFAR-10 dataset.
+
+Support single-host training with one or multiple devices.
+
+ResNet as proposed in:
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+Deep Residual Learning for Image Recognition. arXiv:1512.03385
+
+CIFAR-10 as in:
+http://www.cs.toronto.edu/~kriz/cifar.html
+
+
+"""
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import functools
+import itertools
+import os
+
+import cifar10
+import cifar10_model
+import cifar10_utils
+import numpy as np
+import six
+from six.moves import xrange # pylint: disable=redefined-builtin
+import tensorflow as tf
+
+tf.logging.set_verbosity(tf.logging.INFO)
+
+
+def get_model_fn(num_gpus, variable_strategy, num_workers):
+ """Returns a function that will build the resnet model."""
+
+ def _resnet_model_fn(features, labels, mode, params):
+ """Resnet model body.
+
+ Support single host, one or more GPU training. Parameter distribution can
+ be either one of the following scheme.
+ 1. CPU is the parameter server and manages gradient updates.
+ 2. Parameters are distributed evenly across all GPUs, and the first GPU
+ manages gradient updates.
+
+ Args:
+ features: a list of tensors, one for each tower
+ labels: a list of tensors, one for each tower
+ mode: ModeKeys.TRAIN or EVAL
+ params: Hyperparameters suitable for tuning
+ Returns:
+ A EstimatorSpec object.
+ """
+ is_training = (mode == tf.estimator.ModeKeys.TRAIN)
+ weight_decay = params.weight_decay
+ momentum = params.momentum
+
+ tower_features = features
+ tower_labels = labels
+ tower_losses = []
+ tower_gradvars = []
+ tower_preds = []
+
+ # channels first (NCHW) is normally optimal on GPU and channels last (NHWC)
+ # on CPU. The exception is Intel MKL on CPU which is optimal with
+ # channels_last.
+ data_format = params.data_format
+ if not data_format:
+ if num_gpus == 0:
+ data_format = 'channels_last'
+ else:
+ data_format = 'channels_first'
+
+ if num_gpus == 0:
+ num_devices = 1
+ device_type = 'cpu'
+ else:
+ num_devices = num_gpus
+ device_type = 'gpu'
+
+ for i in range(num_devices):
+ worker_device = '/{}:{}'.format(device_type, i)
+ if variable_strategy == 'CPU':
+ device_setter = cifar10_utils.local_device_setter(
+ worker_device=worker_device)
+ elif variable_strategy == 'GPU':
+ device_setter = cifar10_utils.local_device_setter(
+ ps_device_type='gpu',
+ worker_device=worker_device,
+ ps_strategy=tf.contrib.training.GreedyLoadBalancingStrategy(
+ num_gpus, tf.contrib.training.byte_size_load_fn))
+ with tf.variable_scope('resnet', reuse=bool(i != 0)):
+ with tf.name_scope('tower_%d' % i) as name_scope:
+ with tf.device(device_setter):
+ loss, gradvars, preds = _tower_fn(
+ is_training, weight_decay, tower_features[i], tower_labels[i],
+ data_format, params.num_layers, params.batch_norm_decay,
+ params.batch_norm_epsilon)
+ tower_losses.append(loss)
+ tower_gradvars.append(gradvars)
+ tower_preds.append(preds)
+ if i == 0:
+ # Only trigger batch_norm moving mean and variance update from
+ # the 1st tower. Ideally, we should grab the updates from all
+ # towers but these stats accumulate extremely fast so we can
+ # ignore the other stats from the other towers without
+ # significant detriment.
+ update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
+ name_scope)
+
+ # Now compute global loss and gradients.
+ gradvars = []
+ with tf.name_scope('gradient_averaging'):
+ all_grads = {}
+ for grad, var in itertools.chain(*tower_gradvars):
+ if grad is not None:
+ all_grads.setdefault(var, []).append(grad)
+ for var, grads in six.iteritems(all_grads):
+ # Average gradients on the same device as the variables
+ # to which they apply.
+ with tf.device(var.device):
+ if len(grads) == 1:
+ avg_grad = grads[0]
+ else:
+ avg_grad = tf.multiply(tf.add_n(grads), 1. / len(grads))
+ gradvars.append((avg_grad, var))
+
+ # Device that runs the ops to apply global gradient updates.
+ consolidation_device = '/gpu:0' if variable_strategy == 'GPU' else '/cpu:0'
+ with tf.device(consolidation_device):
+ # Suggested learning rate scheduling from
+ # https://github.com/ppwwyyxx/tensorpack/blob/master/examples/ResNet/cifar10-resnet.py#L155
+ num_batches_per_epoch = cifar10.Cifar10DataSet.num_examples_per_epoch(
+ 'train') // (params.train_batch_size * num_workers)
+ boundaries = [
+ num_batches_per_epoch * x
+ for x in np.array([82, 123, 300], dtype=np.int64)
+ ]
+ staged_lr = [params.learning_rate * x for x in [1, 0.1, 0.01, 0.002]]
+
+ learning_rate = tf.train.piecewise_constant(tf.train.get_global_step(),
+ boundaries, staged_lr)
+
+ loss = tf.reduce_mean(tower_losses, name='loss')
+
+ examples_sec_hook = cifar10_utils.ExamplesPerSecondHook(
+ params.train_batch_size, every_n_steps=10)
+
+ tensors_to_log = {'learning_rate': learning_rate, 'loss': loss}
+
+ logging_hook = tf.train.LoggingTensorHook(
+ tensors=tensors_to_log, every_n_iter=100)
+
+ train_hooks = [logging_hook, examples_sec_hook]
+
+ optimizer = tf.train.MomentumOptimizer(
+ learning_rate=learning_rate, momentum=momentum)
+
+ if params.sync:
+ optimizer = tf.train.SyncReplicasOptimizer(
+ optimizer, replicas_to_aggregate=num_workers)
+ sync_replicas_hook = optimizer.make_session_run_hook(params.is_chief)
+ train_hooks.append(sync_replicas_hook)
+
+ # Create single grouped train op
+ train_op = [
+ optimizer.apply_gradients(
+ gradvars, global_step=tf.train.get_global_step())
+ ]
+ train_op.extend(update_ops)
+ train_op = tf.group(*train_op)
+
+ predictions = {
+ 'classes':
+ tf.concat([p['classes'] for p in tower_preds], axis=0),
+ 'probabilities':
+ tf.concat([p['probabilities'] for p in tower_preds], axis=0)
+ }
+ stacked_labels = tf.concat(labels, axis=0)
+ metrics = {
+ 'accuracy':
+ tf.metrics.accuracy(stacked_labels, predictions['classes'])
+ }
+
+ return tf.estimator.EstimatorSpec(
+ mode=mode,
+ predictions=predictions,
+ loss=loss,
+ train_op=train_op,
+ training_hooks=train_hooks,
+ eval_metric_ops=metrics)
+
+ return _resnet_model_fn
+
+
+def _tower_fn(is_training, weight_decay, feature, label, data_format,
+ num_layers, batch_norm_decay, batch_norm_epsilon):
+ """Build computation tower (Resnet).
+
+ Args:
+ is_training: true if is training graph.
+ weight_decay: weight regularization strength, a float.
+ feature: a Tensor.
+ label: a Tensor.
+ data_format: channels_last (NHWC) or channels_first (NCHW).
+ num_layers: number of layers, an int.
+ batch_norm_decay: decay for batch normalization, a float.
+ batch_norm_epsilon: epsilon for batch normalization, a float.
+
+ Returns:
+ A tuple with the loss for the tower, the gradients and parameters, and
+ predictions.
+
+ """
+ model = cifar10_model.ResNetCifar10(
+ num_layers,
+ batch_norm_decay=batch_norm_decay,
+ batch_norm_epsilon=batch_norm_epsilon,
+ is_training=is_training,
+ data_format=data_format)
+ logits = model.forward_pass(feature, input_data_format='channels_last')
+ tower_pred = {
+ 'classes': tf.argmax(input=logits, axis=1),
+ 'probabilities': tf.nn.softmax(logits)
+ }
+
+ tower_loss = tf.losses.sparse_softmax_cross_entropy(
+ logits=logits, labels=label)
+ tower_loss = tf.reduce_mean(tower_loss)
+
+ model_params = tf.trainable_variables()
+ tower_loss += weight_decay * tf.add_n(
+ [tf.nn.l2_loss(v) for v in model_params])
+
+ tower_grad = tf.gradients(tower_loss, model_params)
+
+ return tower_loss, zip(tower_grad, model_params), tower_pred
+
+
+def input_fn(data_dir,
+ subset,
+ num_shards,
+ batch_size,
+ use_distortion_for_training=True):
+ """Create input graph for model.
+
+ Args:
+ data_dir: Directory where TFRecords representing the dataset are located.
+ subset: one of 'train', 'validate' and 'eval'.
+ num_shards: num of towers participating in data-parallel training.
+ batch_size: total batch size for training to be divided by the number of
+ shards.
+ use_distortion_for_training: True to use distortions.
+ Returns:
+ two lists of tensors for features and labels, each of num_shards length.
+ """
+ with tf.device('/cpu:0'):
+ use_distortion = subset == 'train' and use_distortion_for_training
+ dataset = cifar10.Cifar10DataSet(data_dir, subset, use_distortion)
+ image_batch, label_batch = dataset.make_batch(batch_size)
+ if num_shards <= 1:
+ # No GPU available or only 1 GPU.
+ return [image_batch], [label_batch]
+
+ # Note that passing num=batch_size is safe here, even though
+ # dataset.batch(batch_size) can, in some cases, return fewer than batch_size
+ # examples. This is because it does so only when repeating for a limited
+ # number of epochs, but our dataset repeats forever.
+ image_batch = tf.unstack(image_batch, num=batch_size, axis=0)
+ label_batch = tf.unstack(label_batch, num=batch_size, axis=0)
+ feature_shards = [[] for i in range(num_shards)]
+ label_shards = [[] for i in range(num_shards)]
+ for i in xrange(batch_size):
+ idx = i % num_shards
+ feature_shards[idx].append(image_batch[i])
+ label_shards[idx].append(label_batch[i])
+ feature_shards = [tf.parallel_stack(x) for x in feature_shards]
+ label_shards = [tf.parallel_stack(x) for x in label_shards]
+ return feature_shards, label_shards
+
+
+def get_experiment_fn(data_dir,
+ num_gpus,
+ variable_strategy,
+ use_distortion_for_training=True):
+ """Returns an Experiment function.
+
+ Experiments perform training on several workers in parallel,
+ in other words experiments know how to invoke train and eval in a sensible
+ fashion for distributed training. Arguments passed directly to this
+ function are not tunable, all other arguments should be passed within
+ tf.HParams, passed to the enclosed function.
+
+ Args:
+ data_dir: str. Location of the data for input_fns.
+ num_gpus: int. Number of GPUs on each worker.
+ variable_strategy: String. CPU to use CPU as the parameter server
+ and GPU to use the GPUs as the parameter server.
+ use_distortion_for_training: bool. See cifar10.Cifar10DataSet.
+ Returns:
+ A function (tf.estimator.RunConfig, tf.contrib.training.HParams) ->
+ tf.contrib.learn.Experiment.
+
+ Suitable for use by tf.contrib.learn.learn_runner, which will run various
+ methods on Experiment (train, evaluate) based on information
+ about the current runner in `run_config`.
+ """
+
+ def _experiment_fn(run_config, hparams):
+ """Returns an Experiment."""
+ # Create estimator.
+ train_input_fn = functools.partial(
+ input_fn,
+ data_dir,
+ subset='train',
+ num_shards=num_gpus,
+ batch_size=hparams.train_batch_size,
+ use_distortion_for_training=use_distortion_for_training)
+
+ eval_input_fn = functools.partial(
+ input_fn,
+ data_dir,
+ subset='eval',
+ batch_size=hparams.eval_batch_size,
+ num_shards=num_gpus)
+
+ num_eval_examples = cifar10.Cifar10DataSet.num_examples_per_epoch('eval')
+ if num_eval_examples % hparams.eval_batch_size != 0:
+ raise ValueError(
+ 'validation set size must be multiple of eval_batch_size')
+
+ train_steps = hparams.train_steps
+ eval_steps = num_eval_examples // hparams.eval_batch_size
+
+ classifier = tf.estimator.Estimator(
+ model_fn=get_model_fn(num_gpus, variable_strategy,
+ run_config.num_worker_replicas or 1),
+ config=run_config,
+ params=hparams)
+
+ # Create experiment.
+ return tf.contrib.learn.Experiment(
+ classifier,
+ train_input_fn=train_input_fn,
+ eval_input_fn=eval_input_fn,
+ train_steps=train_steps,
+ eval_steps=eval_steps)
+
+ return _experiment_fn
+
+
+def main(job_dir, data_dir, num_gpus, variable_strategy,
+ use_distortion_for_training, log_device_placement, num_intra_threads,
+ **hparams):
+ # The env variable is on deprecation path, default is set to off.
+ os.environ['TF_SYNC_ON_FINISH'] = '0'
+ os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1'
+
+ # Session configuration.
+ sess_config = tf.ConfigProto(
+ allow_soft_placement=True,
+ log_device_placement=log_device_placement,
+ intra_op_parallelism_threads=num_intra_threads,
+ gpu_options=tf.GPUOptions(force_gpu_compatible=True))
+
+ config = cifar10_utils.RunConfig(
+ session_config=sess_config, model_dir=job_dir)
+ tf.contrib.learn.learn_runner.run(
+ get_experiment_fn(data_dir, num_gpus, variable_strategy,
+ use_distortion_for_training),
+ run_config=config,
+ hparams=tf.contrib.training.HParams(
+ is_chief=config.is_chief,
+ **hparams))
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--data-dir',
+ type=str,
+ required=True,
+ help='The directory where the CIFAR-10 input data is stored.')
+ parser.add_argument(
+ '--job-dir',
+ type=str,
+ required=True,
+ help='The directory where the model will be stored.')
+ parser.add_argument(
+ '--variable-strategy',
+ choices=['CPU', 'GPU'],
+ type=str,
+ default='CPU',
+ help='Where to locate variable operations')
+ parser.add_argument(
+ '--num-gpus',
+ type=int,
+ default=1,
+ help='The number of gpus used. Uses only CPU if set to 0.')
+ parser.add_argument(
+ '--num-layers',
+ type=int,
+ default=44,
+ help='The number of layers of the model.')
+ parser.add_argument(
+ '--train-steps',
+ type=int,
+ default=80000,
+ help='The number of steps to use for training.')
+ parser.add_argument(
+ '--train-batch-size',
+ type=int,
+ default=128,
+ help='Batch size for training.')
+ parser.add_argument(
+ '--eval-batch-size',
+ type=int,
+ default=100,
+ help='Batch size for validation.')
+ parser.add_argument(
+ '--momentum',
+ type=float,
+ default=0.9,
+ help='Momentum for MomentumOptimizer.')
+ parser.add_argument(
+ '--weight-decay',
+ type=float,
+ default=2e-4,
+ help='Weight decay for convolutions.')
+ parser.add_argument(
+ '--learning-rate',
+ type=float,
+ default=0.1,
+ help="""\
+ This is the inital learning rate value. The learning rate will decrease
+ during training. For more details check the model_fn implementation in
+ this file.\
+ """)
+ parser.add_argument(
+ '--use-distortion-for-training',
+ type=bool,
+ default=True,
+ help='If doing image distortion for training.')
+ parser.add_argument(
+ '--sync',
+ action='store_true',
+ default=False,
+ help="""\
+ If present when running in a distributed environment will run on sync mode.\
+ """)
+ parser.add_argument(
+ '--num-intra-threads',
+ type=int,
+ default=0,
+ help="""\
+ Number of threads to use for intra-op parallelism. When training on CPU
+ set to 0 to have the system pick the appropriate number or alternatively
+ set it to the number of physical CPU cores.\
+ """)
+ parser.add_argument(
+ '--num-inter-threads',
+ type=int,
+ default=0,
+ help="""\
+ Number of threads to use for inter-op parallelism. If set to 0, the
+ system will pick an appropriate number.\
+ """)
+ parser.add_argument(
+ '--data-format',
+ type=str,
+ default=None,
+ help="""\
+ If not set, the data format best for the training device is used.
+ Allowed values: channels_first (NCHW) channels_last (NHWC).\
+ """)
+ parser.add_argument(
+ '--log-device-placement',
+ action='store_true',
+ default=False,
+ help='Whether to log device placement.')
+ parser.add_argument(
+ '--batch-norm-decay',
+ type=float,
+ default=0.997,
+ help='Decay for batch norm.')
+ parser.add_argument(
+ '--batch-norm-epsilon',
+ type=float,
+ default=1e-5,
+ help='Epsilon for batch norm.')
+ args = parser.parse_args()
+
+ if args.num_gpus > 0:
+ assert tf.test.is_gpu_available(), "Requested GPUs but none found."
+ if args.num_gpus < 0:
+ raise ValueError(
+ 'Invalid GPU count: \"--num-gpus\" must be 0 or a positive integer.')
+ if args.num_gpus == 0 and args.variable_strategy == 'GPU':
+ raise ValueError('num-gpus=0, CPU must be used as parameter server. Set'
+ '--variable-strategy=CPU.')
+ if (args.num_layers - 2) % 6 != 0:
+ raise ValueError('Invalid --num-layers parameter.')
+ if args.num_gpus != 0 and args.train_batch_size % args.num_gpus != 0:
+ raise ValueError('--train-batch-size must be multiple of --num-gpus.')
+ if args.num_gpus != 0 and args.eval_batch_size % args.num_gpus != 0:
+ raise ValueError('--eval-batch-size must be multiple of --num-gpus.')
+
+ main(**vars(args))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_model.py
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_model.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_model.py
new file mode 100644
index 0000000..d67c233
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_model.py
@@ -0,0 +1,80 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model class for Cifar10 Dataset."""
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+import model_base
+
+
+class ResNetCifar10(model_base.ResNet):
+ """Cifar10 model with ResNetV1 and basic residual block."""
+
+ def __init__(self,
+ num_layers,
+ is_training,
+ batch_norm_decay,
+ batch_norm_epsilon,
+ data_format='channels_first'):
+ super(ResNetCifar10, self).__init__(
+ is_training,
+ data_format,
+ batch_norm_decay,
+ batch_norm_epsilon
+ )
+ self.n = (num_layers - 2) // 6
+ # Add one in case label starts with 1. No impact if label starts with 0.
+ self.num_classes = 10 + 1
+ self.filters = [16, 16, 32, 64]
+ self.strides = [1, 2, 2]
+
+ def forward_pass(self, x, input_data_format='channels_last'):
+ """Build the core model within the graph."""
+ if self._data_format != input_data_format:
+ if input_data_format == 'channels_last':
+ # Computation requires channels_first.
+ x = tf.transpose(x, [0, 3, 1, 2])
+ else:
+ # Computation requires channels_last.
+ x = tf.transpose(x, [0, 2, 3, 1])
+
+ # Image standardization.
+ x = x / 128 - 1
+
+ x = self._conv(x, 3, 16, 1)
+ x = self._batch_norm(x)
+ x = self._relu(x)
+
+ # Use basic (non-bottleneck) block and ResNet V1 (post-activation).
+ res_func = self._residual_v1
+
+ # 3 stages of block stacking.
+ for i in range(3):
+ with tf.name_scope('stage'):
+ for j in range(self.n):
+ if j == 0:
+ # First block in a stage, filters and strides may change.
+ x = res_func(x, 3, self.filters[i], self.filters[i + 1],
+ self.strides[i])
+ else:
+ # Following blocks in a stage, constant filters and unit stride.
+ x = res_func(x, 3, self.filters[i + 1], self.filters[i + 1], 1)
+
+ x = self._global_avg_pool(x)
+ x = self._fully_connected(x, self.num_classes)
+
+ return x
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_utils.py
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_utils.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_utils.py
new file mode 100644
index 0000000..7ecb50a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/cifar10_utils.py
@@ -0,0 +1,154 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import collections
+import six
+
+import tensorflow as tf
+
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.core.framework import node_def_pb2
+from tensorflow.python.framework import device as pydev
+from tensorflow.python.training import basic_session_run_hooks
+from tensorflow.python.training import session_run_hook
+from tensorflow.python.training import training_util
+from tensorflow.python.training import device_setter
+from tensorflow.contrib.learn.python.learn import run_config
+
+
+# TODO(b/64848083) Remove once uid bug is fixed
+class RunConfig(tf.contrib.learn.RunConfig):
+ def uid(self, whitelist=None):
+ """Generates a 'Unique Identifier' based on all internal fields.
+ Caller should use the uid string to check `RunConfig` instance integrity
+ in one session use, but should not rely on the implementation details, which
+ is subject to change.
+ Args:
+ whitelist: A list of the string names of the properties uid should not
+ include. If `None`, defaults to `_DEFAULT_UID_WHITE_LIST`, which
+ includes most properties user allowes to change.
+ Returns:
+ A uid string.
+ """
+ if whitelist is None:
+ whitelist = run_config._DEFAULT_UID_WHITE_LIST
+
+ state = {k: v for k, v in self.__dict__.items() if not k.startswith('__')}
+ # Pop out the keys in whitelist.
+ for k in whitelist:
+ state.pop('_' + k, None)
+
+ ordered_state = collections.OrderedDict(
+ sorted(state.items(), key=lambda t: t[0]))
+ # For class instance without __repr__, some special cares are required.
+ # Otherwise, the object address will be used.
+ if '_cluster_spec' in ordered_state:
+ ordered_state['_cluster_spec'] = collections.OrderedDict(
+ sorted(ordered_state['_cluster_spec'].as_dict().items(),
+ key=lambda t: t[0])
+ )
+ return ', '.join(
+ '%s=%r' % (k, v) for (k, v) in six.iteritems(ordered_state))
+
+
+class ExamplesPerSecondHook(session_run_hook.SessionRunHook):
+ """Hook to print out examples per second.
+
+ Total time is tracked and then divided by the total number of steps
+ to get the average step time and then batch_size is used to determine
+ the running average of examples per second. The examples per second for the
+ most recent interval is also logged.
+ """
+
+ def __init__(
+ self,
+ batch_size,
+ every_n_steps=100,
+ every_n_secs=None,):
+ """Initializer for ExamplesPerSecondHook.
+
+ Args:
+ batch_size: Total batch size used to calculate examples/second from
+ global time.
+ every_n_steps: Log stats every n steps.
+ every_n_secs: Log stats every n seconds.
+ """
+ if (every_n_steps is None) == (every_n_secs is None):
+ raise ValueError('exactly one of every_n_steps'
+ ' and every_n_secs should be provided.')
+ self._timer = basic_session_run_hooks.SecondOrStepTimer(
+ every_steps=every_n_steps, every_secs=every_n_secs)
+
+ self._step_train_time = 0
+ self._total_steps = 0
+ self._batch_size = batch_size
+
+ def begin(self):
+ self._global_step_tensor = training_util.get_global_step()
+ if self._global_step_tensor is None:
+ raise RuntimeError(
+ 'Global step should be created to use StepCounterHook.')
+
+ def before_run(self, run_context): # pylint: disable=unused-argument
+ return basic_session_run_hooks.SessionRunArgs(self._global_step_tensor)
+
+ def after_run(self, run_context, run_values):
+ _ = run_context
+
+ global_step = run_values.results
+ if self._timer.should_trigger_for_step(global_step):
+ elapsed_time, elapsed_steps = self._timer.update_last_triggered_step(
+ global_step)
+ if elapsed_time is not None:
+ steps_per_sec = elapsed_steps / elapsed_time
+ self._step_train_time += elapsed_time
+ self._total_steps += elapsed_steps
+
+ average_examples_per_sec = self._batch_size * (
+ self._total_steps / self._step_train_time)
+ current_examples_per_sec = steps_per_sec * self._batch_size
+ # Average examples/sec followed by current examples/sec
+ logging.info('%s: %g (%g), step = %g', 'Average examples/sec',
+ average_examples_per_sec, current_examples_per_sec,
+ self._total_steps)
+
+def local_device_setter(num_devices=1,
+ ps_device_type='cpu',
+ worker_device='/cpu:0',
+ ps_ops=None,
+ ps_strategy=None):
+ if ps_ops == None:
+ ps_ops = ['Variable', 'VariableV2', 'VarHandleOp']
+
+ if ps_strategy is None:
+ ps_strategy = device_setter._RoundRobinStrategy(num_devices)
+ if not six.callable(ps_strategy):
+ raise TypeError("ps_strategy must be callable")
+
+ def _local_device_chooser(op):
+ current_device = pydev.DeviceSpec.from_string(op.device or "")
+
+ node_def = op if isinstance(op, node_def_pb2.NodeDef) else op.node_def
+ if node_def.op in ps_ops:
+ ps_device_spec = pydev.DeviceSpec.from_string(
+ '/{}:{}'.format(ps_device_type, ps_strategy(op)))
+
+ ps_device_spec.merge_from(current_device)
+ return ps_device_spec.to_string()
+ else:
+ worker_device_spec = pydev.DeviceSpec.from_string(worker_device or "")
+ worker_device_spec.merge_from(current_device)
+ return worker_device_spec.to_string()
+ return _local_device_chooser
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/generate_cifar10_tfrecords.py
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/generate_cifar10_tfrecords.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/generate_cifar10_tfrecords.py
new file mode 100644
index 0000000..409cee4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/generate_cifar10_tfrecords.py
@@ -0,0 +1,114 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Read CIFAR-10 data from pickled numpy arrays and writes TFRecords.
+
+Generates tf.train.Example protos and writes them to TFRecord files from the
+python version of the CIFAR-10 dataset downloaded from
+https://www.cs.toronto.edu/~kriz/cifar.html.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import os
+
+import tarfile
+from six.moves import cPickle as pickle
+from six.moves import xrange # pylint: disable=redefined-builtin
+import tensorflow as tf
+
+CIFAR_FILENAME = 'cifar-10-python.tar.gz'
+CIFAR_DOWNLOAD_URL = 'https://www.cs.toronto.edu/~kriz/' + CIFAR_FILENAME
+CIFAR_LOCAL_FOLDER = 'cifar-10-batches-py'
+
+
+def download_and_extract(data_dir):
+ # download CIFAR-10 if not already downloaded.
+ tf.contrib.learn.datasets.base.maybe_download(CIFAR_FILENAME, data_dir,
+ CIFAR_DOWNLOAD_URL)
+ tarfile.open(os.path.join(data_dir, CIFAR_FILENAME),
+ 'r:gz').extractall(data_dir)
+
+
+def _int64_feature(value):
+ return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+
+def _bytes_feature(value):
+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+
+def _get_file_names():
+ """Returns the file names expected to exist in the input_dir."""
+ file_names = {}
+ file_names['train'] = ['data_batch_%d' % i for i in xrange(1, 5)]
+ file_names['validation'] = ['data_batch_5']
+ file_names['eval'] = ['test_batch']
+ return file_names
+
+
+def read_pickle_from_file(filename):
+ with tf.gfile.Open(filename, 'rb') as f:
+ data_dict = pickle.load(f)
+ return data_dict
+
+
+def convert_to_tfrecord(input_files, output_file):
+ """Converts a file to TFRecords."""
+ print('Generating %s' % output_file)
+ with tf.python_io.TFRecordWriter(output_file) as record_writer:
+ for input_file in input_files:
+ data_dict = read_pickle_from_file(input_file)
+ data = data_dict['data']
+ labels = data_dict['labels']
+ num_entries_in_batch = len(labels)
+ for i in range(num_entries_in_batch):
+ example = tf.train.Example(features=tf.train.Features(
+ feature={
+ 'image': _bytes_feature(data[i].tobytes()),
+ 'label': _int64_feature(labels[i])
+ }))
+ record_writer.write(example.SerializeToString())
+
+
+def main(data_dir):
+ print('Download from {} and extract.'.format(CIFAR_DOWNLOAD_URL))
+ download_and_extract(data_dir)
+ file_names = _get_file_names()
+ input_dir = os.path.join(data_dir, CIFAR_LOCAL_FOLDER)
+ for mode, files in file_names.items():
+ input_files = [os.path.join(input_dir, f) for f in files]
+ output_file = os.path.join(data_dir, mode + '.tfrecords')
+ try:
+ os.remove(output_file)
+ except OSError:
+ pass
+ # Convert to tf.train.Example and write the to TFRecords.
+ convert_to_tfrecord(input_files, output_file)
+ print('Done!')
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ '--data-dir',
+ type=str,
+ default='',
+ help='Directory to download and extract CIFAR-10 to.')
+
+ args = parser.parse_args()
+ main(args.data_dir)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: Changed version in trunk to
3.3.0-SNAPSHOT.
Posted by in...@apache.org.
Changed version in trunk to 3.3.0-SNAPSHOT.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58fa96b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58fa96b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58fa96b6
Branch: refs/heads/HDFS-13532
Commit: 58fa96b6974411839bd2149071c95df06f3321f5
Parents: a392962
Author: Sunil G <su...@apache.org>
Authored: Tue Oct 2 22:41:41 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Oct 2 22:41:41 2018 +0530
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 4 ++--
hadoop-build-tools/pom.xml | 2 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 4 ++--
.../hadoop-client-check-invariants/pom.xml | 4 ++--
.../hadoop-client-check-test-invariants/pom.xml | 4 ++--
.../hadoop-client-integration-tests/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client-minicluster/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client-runtime/pom.xml | 4 ++--
hadoop-client-modules/hadoop-client/pom.xml | 4 ++--
hadoop-client-modules/pom.xml | 2 +-
hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 4 ++--
hadoop-cloud-storage-project/pom.xml | 4 ++--
hadoop-common-project/hadoop-annotations/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth-examples/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth/pom.xml | 4 ++--
hadoop-common-project/hadoop-common/pom.xml | 4 ++--
hadoop-common-project/hadoop-kms/pom.xml | 4 ++--
hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++--
hadoop-common-project/hadoop-nfs/pom.xml | 4 ++--
hadoop-common-project/pom.xml | 4 ++--
hadoop-dist/pom.xml | 4 ++--
hadoop-hdds/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++--
hadoop-hdfs-project/pom.xml | 4 ++--
.../hadoop-mapreduce-client-app/pom.xml | 4 ++--
.../hadoop-mapreduce-client-common/pom.xml | 4 ++--
.../hadoop-mapreduce-client-core/pom.xml | 4 ++--
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 ++--
.../hadoop-mapreduce-client-hs/pom.xml | 4 ++--
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 ++--
.../hadoop-mapreduce-client-nativetask/pom.xml | 4 ++--
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 ++--
.../hadoop-mapreduce-client-uploader/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++--
.../hadoop-mapreduce-examples/pom.xml | 4 ++--
hadoop-mapreduce-project/pom.xml | 4 ++--
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 ++--
hadoop-ozone/pom.xml | 2 +-
hadoop-project-dist/pom.xml | 4 ++--
hadoop-project/pom.xml | 4 ++--
hadoop-tools/hadoop-aliyun/pom.xml | 2 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 4 ++--
hadoop-tools/hadoop-archives/pom.xml | 4 ++--
hadoop-tools/hadoop-aws/pom.xml | 4 ++--
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
hadoop-tools/hadoop-azure/pom.xml | 2 +-
hadoop-tools/hadoop-datajoin/pom.xml | 4 ++--
hadoop-tools/hadoop-distcp/pom.xml | 4 ++--
hadoop-tools/hadoop-extras/pom.xml | 4 ++--
hadoop-tools/hadoop-fs2img/pom.xml | 4 ++--
hadoop-tools/hadoop-gridmix/pom.xml | 4 ++--
hadoop-tools/hadoop-kafka/pom.xml | 4 ++--
hadoop-tools/hadoop-openstack/pom.xml | 4 ++--
hadoop-tools/hadoop-pipes/pom.xml | 4 ++--
hadoop-tools/hadoop-resourceestimator/pom.xml | 2 +-
hadoop-tools/hadoop-rumen/pom.xml | 4 ++--
hadoop-tools/hadoop-sls/pom.xml | 4 ++--
hadoop-tools/hadoop-streaming/pom.xml | 4 ++--
hadoop-tools/hadoop-tools-dist/pom.xml | 4 ++--
hadoop-tools/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml | 4 ++--
.../hadoop-yarn-applications-distributedshell/pom.xml | 4 ++--
.../pom.xml | 4 ++--
.../hadoop-yarn-services/hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../hadoop-yarn-applications/hadoop-yarn-services/pom.xml | 2 +-
.../hadoop-yarn-submarine/pom.xml | 10 +++++-----
.../hadoop-yarn/hadoop-yarn-applications/pom.xml | 4 ++--
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 ++--
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 ++--
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 ++--
.../hadoop-yarn-server-applicationhistoryservice/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 4 ++--
.../hadoop-yarn-server-nodemanager/pom.xml | 4 ++--
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-router/pom.xml | 4 ++--
.../hadoop-yarn-server-sharedcachemanager/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml | 4 ++--
.../hadoop-yarn-server-timeline-pluginstorage/pom.xml | 4 ++--
.../pom.xml | 4 ++--
.../pom.xml | 2 +-
.../pom.xml | 4 ++--
.../pom.xml | 4 ++--
.../pom.xml | 4 ++--
.../pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice-hbase/pom.xml | 4 ++--
.../hadoop-yarn-server-timelineservice/pom.xml | 4 ++--
.../hadoop-yarn-server-web-proxy/pom.xml | 4 ++--
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 ++--
hadoop-yarn-project/pom.xml | 4 ++--
pom.xml | 2 +-
100 files changed, 189 insertions(+), 189 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-assemblies/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 6979311..aabef34 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -23,11 +23,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-assemblies</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop Assemblies</name>
<description>Apache Hadoop Assemblies</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-build-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-build-tools/pom.xml b/hadoop-build-tools/pom.xml
index 655550e..b94dd32 100644
--- a/hadoop-build-tools/pom.xml
+++ b/hadoop-build-tools/pom.xml
@@ -18,7 +18,7 @@
<parent>
<artifactId>hadoop-main</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-build-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/hadoop-client-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml
index 024f6ba..9e4406d 100644
--- a/hadoop-client-modules/hadoop-client-api/pom.xml
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-api</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
index 3989c30..4319a8b 100644
--- a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-check-invariants</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
index 4f35ba3..70a987f 100644
--- a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-check-test-invariants</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
index 68d2969..558372d 100644
--- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
+++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-integration-tests</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Checks that we can use the generated artifacts</description>
<name>Apache Hadoop Client Packaging Integration Tests</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index 70fca8a..5ffe6f9 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-minicluster</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Minicluster for Clients</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/hadoop-client-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml
index bfa6c15..578ddb5 100644
--- a/hadoop-client-modules/hadoop-client-runtime/pom.xml
+++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-runtime</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
index 19c313e..86b23fe 100644
--- a/hadoop-client-modules/hadoop-client/pom.xml
+++ b/hadoop-client-modules/hadoop-client/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-client</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Client aggregation pom with dependencies exposed</description>
<name>Apache Hadoop Client Aggregator</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-client-modules/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml
index 9ddd526..3273240 100644
--- a/hadoop-client-modules/pom.xml
+++ b/hadoop-client-modules/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-modules</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index f7a5732..16e9582 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-cloud-storage</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Cloud Storage</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-cloud-storage-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cloud-storage-project/pom.xml b/hadoop-cloud-storage-project/pom.xml
index 6ede052..50dd70e 100644
--- a/hadoop-cloud-storage-project/pom.xml
+++ b/hadoop-cloud-storage-project/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-cloud-storage-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Cloud Storage Project</description>
<name>Apache Hadoop Cloud Storage Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/hadoop-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index fd329e2..d8373c8 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-annotations</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Annotations</description>
<name>Apache Hadoop Annotations</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/hadoop-auth-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index f2f4a5d..a105885 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-auth-examples</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>Apache Hadoop Auth Examples</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/hadoop-auth/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 8aae2c7..45d727f 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-auth</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop Auth</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 1e6da92..6d9e296 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-common</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Common</description>
<name>Apache Hadoop Common</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 03a68c5..0ed3dc3 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -22,11 +22,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-kms</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop KMS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/hadoop-minikdc/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml
index c92b6b1..f1566f1 100644
--- a/hadoop-common-project/hadoop-minikdc/pom.xml
+++ b/hadoop-common-project/hadoop-minikdc/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-minikdc</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop MiniKDC</description>
<name>Apache Hadoop MiniKDC</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/hadoop-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 80d8cd2..68c3339 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-nfs</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop NFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-common-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index a1a511f..9f57569 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-common-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Common Project</description>
<name>Apache Hadoop Common Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 5447b6a..1d3db65 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Distribution</description>
<name>Apache Hadoop Distribution</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 59dfa79..f9a6b57 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -20,7 +20,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project-dist</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index a5ed7a3..1a1854d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -20,11 +20,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs-client</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop HDFS Client</description>
<name>Apache Hadoop HDFS Client</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index eaf5c52..c0078da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -22,11 +22,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-hdfs-httpfs</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop HttpFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index ded1c0d..7464b3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -20,11 +20,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs-native-client</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop HDFS Native Client</description>
<name>Apache Hadoop HDFS Native Client</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 42d2c00..8d796ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -20,11 +20,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-hdfs-nfs</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop HDFS-NFS</description>
<name>Apache Hadoop HDFS-NFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index b883726..386eb41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -20,11 +20,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs-rbf</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop HDFS-RBF</description>
<name>Apache Hadoop HDFS-RBF</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 065c175..a0ea04f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -20,11 +20,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-hdfs</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop HDFS</description>
<name>Apache Hadoop HDFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index 0aa1112..b26ced0 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -20,11 +20,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-hdfs-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop HDFS Project</description>
<name>Apache Hadoop HDFS Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index fe0a6ba..0a354c8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-app</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce App</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 5c4b83a..2c6e8e5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index f5bc301..bfac6a9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Core</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
index 715d0f6..1a64912 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce HistoryServer Plugins</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index c174140..b371750 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce HistoryServer</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index aec0fe3..b961cc7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce JobClient</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index c583576..20acbc4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-nativetask</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce NativeTask</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
index d5c7a9a..9711597 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Shuffle</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
index 12ba3f9..5d53592 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
@@ -18,11 +18,11 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-mapreduce-client-uploader</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Uploader</name>
<dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index 430116b..e22cf77 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-mapreduce-client</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop MapReduce Client</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index 7f2fb8f..39c50e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-mapreduce-examples</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Examples</description>
<name>Apache Hadoop MapReduce Examples</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index fe89729..a848fc0 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-mapreduce</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Hadoop MapReduce</name>
<url>http://hadoop.apache.org/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index d6b18b4..16bad69 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-maven-plugins</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index 5a35365..34f5b0e 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-minicluster</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Mini-Cluster</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index bb96523..ff6a143 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -20,7 +20,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-ozone</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index fd67a71..2abd359 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Project Dist POM</description>
<name>Apache Hadoop Project Dist POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index e7baee0..cd38376 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -20,10 +20,10 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-aliyun/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index ed60783..13e1296 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-aliyun</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-archive-logs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml
index 3ec3cfd..4f17d29 100644
--- a/hadoop-tools/hadoop-archive-logs/pom.xml
+++ b/hadoop-tools/hadoop-archive-logs/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-archive-logs</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Archive Logs</description>
<name>Apache Hadoop Archive Logs</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-archives/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml
index cfd9790..bf0fb5e 100644
--- a/hadoop-tools/hadoop-archives/pom.xml
+++ b/hadoop-tools/hadoop-archives/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-archives</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Archives</description>
<name>Apache Hadoop Archives</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 78a3d5e..a7087d4 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-aws</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop Amazon Web Services support</name>
<description>
This module contains code to support integration with Amazon Web Services.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 5603db9..4c7edc6 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-azure-datalake</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index 5de9875..c396858 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-azure</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-datajoin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml
index 1f0a151..ba23b43 100644
--- a/hadoop-tools/hadoop-datajoin/pom.xml
+++ b/hadoop-tools/hadoop-datajoin/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-datajoin</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Data Join</description>
<name>Apache Hadoop Data Join</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-distcp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml
index 7fee80b..eccdaac 100644
--- a/hadoop-tools/hadoop-distcp/pom.xml
+++ b/hadoop-tools/hadoop-distcp/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-distcp</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Distributed Copy</description>
<name>Apache Hadoop Distributed Copy</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-extras/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml
index 606b7d7..270696a 100644
--- a/hadoop-tools/hadoop-extras/pom.xml
+++ b/hadoop-tools/hadoop-extras/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-extras</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Extras</description>
<name>Apache Hadoop Extras</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-fs2img/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/pom.xml b/hadoop-tools/hadoop-fs2img/pom.xml
index 747e0b1..8588eee 100644
--- a/hadoop-tools/hadoop-fs2img/pom.xml
+++ b/hadoop-tools/hadoop-fs2img/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-fs2img</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Image Generation Tool</description>
<name>Apache Hadoop Image Generation Tool</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-gridmix/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml
index 8625291..857d56c 100644
--- a/hadoop-tools/hadoop-gridmix/pom.xml
+++ b/hadoop-tools/hadoop-gridmix/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-gridmix</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Gridmix</description>
<name>Apache Hadoop Gridmix</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml
index 1a3443a..9d79c2c 100644
--- a/hadoop-tools/hadoop-kafka/pom.xml
+++ b/hadoop-tools/hadoop-kafka/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-kafka</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop Kafka Library support</name>
<description>
This module contains code to support integration with Kafka.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-openstack/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index b840cc3..466b01b 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-openstack</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop OpenStack support</name>
<description>
This module contains code to support integration with OpenStack.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-pipes/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 367a3d8..2ce1a9b 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-pipes</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Pipes</description>
<name>Apache Hadoop Pipes</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-resourceestimator/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml
index 182264b..2c0c091 100644
--- a/hadoop-tools/hadoop-resourceestimator/pom.xml
+++ b/hadoop-tools/hadoop-resourceestimator/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-resourceestimator</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-rumen/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index 9608717..15bbae7 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-rumen</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Rumen</description>
<name>Apache Hadoop Rumen</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-sls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index a42f8de..d98d784 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-sls</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Scheduler Load Simulator</description>
<name>Apache Hadoop Scheduler Load Simulator</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml
index c4c916a..1ab024e 100644
--- a/hadoop-tools/hadoop-streaming/pom.xml
+++ b/hadoop-tools/hadoop-streaming/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-streaming</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Streaming</description>
<name>Apache Hadoop MapReduce Streaming</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 42ce94c..0deae88 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-tools-dist</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Tools Dist</description>
<name>Apache Hadoop Tools Dist</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index dca59d3..655b5a5 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-tools</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<description>Apache Hadoop Tools</description>
<name>Apache Hadoop Tools</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index 3994104..4f11ad6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-api</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN API</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 7cb7ac7..bc160c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-applications-distributedshell</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN DistributedShell</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index d296c16..478262c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-applications-unmanaged-am-launcher</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Unmanaged Am Launcher</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
index 7386e41..42edbbb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-services-api</artifactId>
<name>Apache Hadoop YARN Services API</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 80a03f9..c23c2bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-services-core</artifactId>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
index 51e19b6..5b50266 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-services</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/pom.xml
index 90a1a6c..a0282ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-submarine</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Yet Another Learning Platform</name>
<properties>
@@ -161,17 +161,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-fs2img</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services-api</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services-core</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 4c03f3c..a1ad4c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-applications</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Applications</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index 913c383..2e0c777 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -17,10 +17,10 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-client</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Client</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 685eac9..de0d78b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-common</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index 4f09190..6a7d202 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-registry</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Registry</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 0527095..5ea8c9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN ApplicationHistoryService</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 52a1c0e..18edc53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-common</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Server Common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 74be4da..52ef240 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN NodeManager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 72bcd91..399aed8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN ResourceManager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
index 22b9b49..5cd2744 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-router</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Router</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
index 65395bc..fa819db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
@@ -17,10 +17,10 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-server-sharedcachemanager</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN SharedCacheManager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 8329fd7..8c559b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -19,10 +19,10 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Server Tests</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
index c50b575..725d8c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timeline-pluginstorage</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN Timeline Plugin Storage</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index 05a5c65..f1ca3ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -22,11 +22,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-tests</artifactId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
<name>Apache Hadoop YARN TimelineService HBase tests</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58fa96b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
index 839b9a0..86b2158 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.2.0-SNAPSHOT</version>
+ <version>3.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase-client</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-8785. Improve the error
message when a bind mount is not whitelisted. Contributed by Simon Prewo
Posted by in...@apache.org.
YARN-8785. Improve the error message when a bind mount is not whitelisted. Contributed by Simon Prewo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5edb9d3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5edb9d3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5edb9d3b
Branch: refs/heads/HDFS-13532
Commit: 5edb9d3b972ad51e23b9cac7928b38876068a463
Parents: 6fab688
Author: Shane Kumpf <sk...@apache.org>
Authored: Tue Oct 2 07:16:29 2018 -0600
Committer: Shane Kumpf <sk...@apache.org>
Committed: Tue Oct 2 07:16:29 2018 -0600
----------------------------------------------------------------------
.../src/main/native/container-executor/impl/utils/docker-util.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5edb9d3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index ee80798..69f27ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -1225,7 +1225,7 @@ static int add_mounts(const struct configuration *command_config, const struct c
if (strncmp("rw", mount_type, 2) == 0) {
// rw mount
if (permitted_rw == 0) {
- fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ fprintf(ERRORFILE, "Configuration does not allow docker mount '%s', realpath=%s\n", values[i], mount_src);
ret = INVALID_DOCKER_RW_MOUNT;
goto free_and_exit;
} else {
@@ -1245,7 +1245,7 @@ static int add_mounts(const struct configuration *command_config, const struct c
} else {
// ro mount
if (permitted_ro == 0 && permitted_rw == 0) {
- fprintf(ERRORFILE, "Invalid docker ro mount '%s', realpath=%s\n", values[i], mount_src);
+ fprintf(ERRORFILE, "Configuration does not allow docker mount '%s', realpath=%s\n", values[i], mount_src);
ret = INVALID_DOCKER_RO_MOUNT;
goto free_and_exit;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-8824. App Nodelabel missed
after RM restart for finished apps. Contributed by Bibin A Chundatt.
Posted by in...@apache.org.
YARN-8824. App Nodelabel missed after RM restart for finished apps. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5287a4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5287a4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5287a4f
Branch: refs/heads/HDFS-13532
Commit: e5287a4fe0bb03d929f066fc50eb0e7bd74bb759
Parents: 3f61950
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Sep 26 12:30:26 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Sep 26 12:30:26 2018 +0530
----------------------------------------------------------------------
.../resourcemanager/recovery/RMStateStore.java | 1 +
.../resourcemanager/MockMemoryRMStateStore.java | 1 +
.../yarn/server/resourcemanager/MockRM.java | 31 +++++++++++++
.../server/resourcemanager/TestRMRestart.java | 49 ++++++++++++++++++++
4 files changed, 82 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index a0b10b1..ccd6fc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -307,6 +307,7 @@ public abstract class RMStateStore extends AbstractService {
context.setApplicationTags(srcCtx.getApplicationTags());
context.setApplicationType(srcCtx.getApplicationType());
context.setUnmanagedAM(srcCtx.getUnmanagedAM());
+ context.setNodeLabelExpression(srcCtx.getNodeLabelExpression());
ContainerLaunchContextPBImpl amContainerSpec =
new ContainerLaunchContextPBImpl();
amContainerSpec.setApplicationACLs(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
index 698f1c6..ef9606f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
@@ -105,6 +105,7 @@ public class MockMemoryRMStateStore extends MemoryRMStateStore {
oldAppSubCtxt.getAMContainerResourceRequests());
context.setLogAggregationContext(oldAppSubCtxt.getLogAggregationContext());
context.setApplicationType(oldAppSubCtxt.getApplicationType());
+ context.setNodeLabelExpression(oldAppSubCtxt.getNodeLabelExpression());
this.appSubCtxtCopy.put(oldAppSubCtxt.getApplicationId(), context);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 6a024f5..df86f28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -722,6 +722,17 @@ public class MockRM extends ResourceManager {
amResourceRequests.get(0).getNodeLabelExpression(), null, null);
}
+ public RMApp submitApp(List<ResourceRequest> amResourceRequests,
+ String appNodeLabel) throws Exception {
+ return submitApp(amResourceRequests, "app1", "user", null, false, null,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true,
+ false, false, null, 0, null, true,
+ amResourceRequests.get(0).getPriority(),
+ amResourceRequests.get(0).getNodeLabelExpression(), null, null, null,
+ appNodeLabel);
+ }
+
public RMApp submitApp(Resource capability, String name, String user,
Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
int maxAppAttempts, Credentials ts, String appType,
@@ -771,6 +782,23 @@ public class MockRM extends ResourceManager {
boolean cancelTokensWhenComplete, Priority priority, String amLabel,
Map<ApplicationTimeoutType, Long> applicationTimeouts,
ByteBuffer tokensConf, Set<String> applicationTags) throws Exception {
+ return submitApp(amResourceRequests, name, user, acls, unmanaged, queue,
+ maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
+ isAppIdProvided, applicationId, attemptFailuresValidityInterval,
+ logAggregationContext, cancelTokensWhenComplete, priority, amLabel,
+ applicationTimeouts, tokensConf, applicationTags, null);
+ }
+
+ public RMApp submitApp(List<ResourceRequest> amResourceRequests, String name,
+ String user, Map<ApplicationAccessType, String> acls, boolean unmanaged,
+ String queue, int maxAppAttempts, Credentials ts, String appType,
+ boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
+ ApplicationId applicationId, long attemptFailuresValidityInterval,
+ LogAggregationContext logAggregationContext,
+ boolean cancelTokensWhenComplete, Priority priority, String amLabel,
+ Map<ApplicationTimeoutType, Long> applicationTimeouts,
+ ByteBuffer tokensConf, Set<String> applicationTags, String appNodeLabel)
+ throws Exception {
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = getClientRMService();
if (! isAppIdProvided) {
@@ -801,6 +829,9 @@ public class MockRM extends ResourceManager {
if (priority != null) {
sub.setPriority(priority);
}
+ if (appNodeLabel != null) {
+ sub.setNodeLabelExpression(appNodeLabel);
+ }
sub.setApplicationType(appType);
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index a8b8d89..d5880ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -88,6 +89,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
@@ -457,6 +459,49 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
}
@Test(timeout = 60000)
+ public void testAppReportNodeLabelRMRestart() throws Exception {
+ if (getSchedulerType() != SchedulerType.CAPACITY) {
+ return;
+ }
+ // Create RM
+ YarnConfiguration newConf = new YarnConfiguration(conf);
+ newConf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ MockRM rm1 = createMockRM(newConf);
+ NodeLabel amLabel = NodeLabel.newInstance("AMLABEL");
+ NodeLabel appLabel = NodeLabel.newInstance("APPLABEL");
+ List<NodeLabel> labels = new ArrayList<>();
+ labels.add(amLabel);
+ labels.add(appLabel);
+ MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ rm1.start();
+ // Add label
+ rm1.getAdminService().addToClusterNodeLabels(
+ AddToClusterNodeLabelsRequest.newInstance(labels));
+ // create app and launch the AM
+ ResourceRequest amResourceRequest = ResourceRequest
+ .newInstance(Priority.newInstance(0), ResourceRequest.ANY,
+ Resource.newInstance(200, 1), 1, true, amLabel.getName());
+ ArrayList resReqs = new ArrayList<>();
+ resReqs.add(amResourceRequest);
+ RMApp app0 = rm1.submitApp(resReqs, appLabel.getName());
+ rm1.killApp(app0.getApplicationId());
+ rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED);
+ // start new RM
+ MockRM rm2 = createMockRM(conf, memStore);
+ rm2.start();
+ Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
+ ApplicationReport appReport = rm2.getClientRMService().getApplicationReport(
+ GetApplicationReportRequest.newInstance(app0.getApplicationId()))
+ .getApplicationReport();
+ Assert
+ .assertEquals(amLabel.getName(), appReport.getAmNodeLabelExpression());
+ Assert.assertEquals(appLabel.getName(),
+ appReport.getAppNodeLabelExpression());
+ rm1.stop();
+ rm2.stop();
+ }
+
+ @Test(timeout = 60000)
public void testUnManagedRMRestart() throws Exception {
// Create RM
MockRM rm1 = createMockRM(conf);
@@ -471,6 +516,10 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
MockRM rm2 = createMockRM(conf, memStore);
rm2.start();
Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
+ ApplicationReport appReport = rm2.getClientRMService().getApplicationReport(
+ GetApplicationReportRequest.newInstance(app0.getApplicationId()))
+ .getApplicationReport();
+ Assert.assertEquals(true, appReport.isUnmanagedApp());
rm1.stop();
rm2.stop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-8800. Updated documentation
of Submarine with latest examples. Contributed by Wangda Tan.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/model_base.py
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/model_base.py b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/model_base.py
new file mode 100644
index 0000000..35e52b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/with-cifar10-models/ubuntu-16.04/cifar10_estimator_tf_1.8.0/model_base.py
@@ -0,0 +1,219 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ResNet model.
+
+Related papers:
+https://arxiv.org/pdf/1603.05027v2.pdf
+https://arxiv.org/pdf/1512.03385v1.pdf
+https://arxiv.org/pdf/1605.07146v1.pdf
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+
+class ResNet(object):
+ """ResNet model."""
+
+ def __init__(self, is_training, data_format, batch_norm_decay, batch_norm_epsilon):
+ """ResNet constructor.
+
+ Args:
+ is_training: if build training or inference model.
+ data_format: the data_format used during computation.
+ one of 'channels_first' or 'channels_last'.
+ """
+ self._batch_norm_decay = batch_norm_decay
+ self._batch_norm_epsilon = batch_norm_epsilon
+ self._is_training = is_training
+ assert data_format in ('channels_first', 'channels_last')
+ self._data_format = data_format
+
+ def forward_pass(self, x):
+ raise NotImplementedError(
+ 'forward_pass() is implemented in ResNet sub classes')
+
+ def _residual_v1(self,
+ x,
+ kernel_size,
+ in_filter,
+ out_filter,
+ stride,
+ activate_before_residual=False):
+ """Residual unit with 2 sub layers, using Plan A for shortcut connection."""
+
+ del activate_before_residual
+ with tf.name_scope('residual_v1') as name_scope:
+ orig_x = x
+
+ x = self._conv(x, kernel_size, out_filter, stride)
+ x = self._batch_norm(x)
+ x = self._relu(x)
+
+ x = self._conv(x, kernel_size, out_filter, 1)
+ x = self._batch_norm(x)
+
+ if in_filter != out_filter:
+ orig_x = self._avg_pool(orig_x, stride, stride)
+ pad = (out_filter - in_filter) // 2
+ if self._data_format == 'channels_first':
+ orig_x = tf.pad(orig_x, [[0, 0], [pad, pad], [0, 0], [0, 0]])
+ else:
+ orig_x = tf.pad(orig_x, [[0, 0], [0, 0], [0, 0], [pad, pad]])
+
+ x = self._relu(tf.add(x, orig_x))
+
+ tf.logging.info('image after unit %s: %s', name_scope, x.get_shape())
+ return x
+
+ def _residual_v2(self,
+ x,
+ in_filter,
+ out_filter,
+ stride,
+ activate_before_residual=False):
+ """Residual unit with 2 sub layers with preactivation, plan A shortcut."""
+
+ with tf.name_scope('residual_v2') as name_scope:
+ if activate_before_residual:
+ x = self._batch_norm(x)
+ x = self._relu(x)
+ orig_x = x
+ else:
+ orig_x = x
+ x = self._batch_norm(x)
+ x = self._relu(x)
+
+ x = self._conv(x, 3, out_filter, stride)
+
+ x = self._batch_norm(x)
+ x = self._relu(x)
+ x = self._conv(x, 3, out_filter, [1, 1, 1, 1])
+
+ if in_filter != out_filter:
+ pad = (out_filter - in_filter) // 2
+ orig_x = self._avg_pool(orig_x, stride, stride)
+ if self._data_format == 'channels_first':
+ orig_x = tf.pad(orig_x, [[0, 0], [pad, pad], [0, 0], [0, 0]])
+ else:
+ orig_x = tf.pad(orig_x, [[0, 0], [0, 0], [0, 0], [pad, pad]])
+
+ x = tf.add(x, orig_x)
+
+ tf.logging.info('image after unit %s: %s', name_scope, x.get_shape())
+ return x
+
+ def _bottleneck_residual_v2(self,
+ x,
+ in_filter,
+ out_filter,
+ stride,
+ activate_before_residual=False):
+ """Bottleneck residual unit with 3 sub layers, plan B shortcut."""
+
+ with tf.name_scope('bottle_residual_v2') as name_scope:
+ if activate_before_residual:
+ x = self._batch_norm(x)
+ x = self._relu(x)
+ orig_x = x
+ else:
+ orig_x = x
+ x = self._batch_norm(x)
+ x = self._relu(x)
+
+ x = self._conv(x, 1, out_filter // 4, stride, is_atrous=True)
+
+ x = self._batch_norm(x)
+ x = self._relu(x)
+ # pad when stride isn't unit
+ x = self._conv(x, 3, out_filter // 4, 1, is_atrous=True)
+
+ x = self._batch_norm(x)
+ x = self._relu(x)
+ x = self._conv(x, 1, out_filter, 1, is_atrous=True)
+
+ if in_filter != out_filter:
+ orig_x = self._conv(orig_x, 1, out_filter, stride, is_atrous=True)
+ x = tf.add(x, orig_x)
+
+ tf.logging.info('image after unit %s: %s', name_scope, x.get_shape())
+ return x
+
+ def _conv(self, x, kernel_size, filters, strides, is_atrous=False):
+ """Convolution."""
+
+ padding = 'SAME'
+ if not is_atrous and strides > 1:
+ pad = kernel_size - 1
+ pad_beg = pad // 2
+ pad_end = pad - pad_beg
+ if self._data_format == 'channels_first':
+ x = tf.pad(x, [[0, 0], [0, 0], [pad_beg, pad_end], [pad_beg, pad_end]])
+ else:
+ x = tf.pad(x, [[0, 0], [pad_beg, pad_end], [pad_beg, pad_end], [0, 0]])
+ padding = 'VALID'
+ return tf.layers.conv2d(
+ inputs=x,
+ kernel_size=kernel_size,
+ filters=filters,
+ strides=strides,
+ padding=padding,
+ use_bias=False,
+ data_format=self._data_format)
+
+ def _batch_norm(self, x):
+ if self._data_format == 'channels_first':
+ data_format = 'NCHW'
+ else:
+ data_format = 'NHWC'
+ return tf.contrib.layers.batch_norm(
+ x,
+ decay=self._batch_norm_decay,
+ center=True,
+ scale=True,
+ epsilon=self._batch_norm_epsilon,
+ is_training=self._is_training,
+ fused=True,
+ data_format=data_format)
+
+ def _relu(self, x):
+ return tf.nn.relu(x)
+
+ def _fully_connected(self, x, out_dim):
+ with tf.name_scope('fully_connected') as name_scope:
+ x = tf.layers.dense(x, out_dim)
+
+ tf.logging.info('image after unit %s: %s', name_scope, x.get_shape())
+ return x
+
+ def _avg_pool(self, x, pool_size, stride):
+ with tf.name_scope('avg_pool') as name_scope:
+ x = tf.layers.average_pooling2d(
+ x, pool_size, stride, 'SAME', data_format=self._data_format)
+
+ tf.logging.info('image after unit %s: %s', name_scope, x.get_shape())
+ return x
+
+ def _global_avg_pool(self, x):
+ with tf.name_scope('global_avg_pool') as name_scope:
+ assert x.get_shape().ndims == 4
+ if self._data_format == 'channels_first':
+ x = tf.reduce_mean(x, [2, 3])
+ else:
+ x = tf.reduce_mean(x, [1, 2])
+ tf.logging.info('image after unit %s: %s', name_scope, x.get_shape())
+ return x
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/Dockerfile.gpu
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/Dockerfile.gpu b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/Dockerfile.gpu
new file mode 100644
index 0000000..05d5fe7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/Dockerfile.gpu
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM nvidia/cuda:9.0-base-ubuntu16.04
+
+RUN echo "$LOG_TAG update and install basic packages" && \
+ apt-get -y update && apt-get install -y --no-install-recommends \
+ build-essential \
+ curl \
+ libfreetype6-dev \
+ libpng12-dev \
+ libzmq3-dev \
+ pkg-config \
+ rsync \
+ software-properties-common \
+ unzip \
+ vim \
+ wget \
+ && \
+ apt-get install -y locales && \
+ locale-gen $LANG && \
+ apt-get clean && \
+ apt -y autoclean && \
+ apt -y dist-upgrade && \
+ apt-get install -y build-essential && \
+ rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+RUN echo "$LOG_TAG Install java8" && \
+ apt-get -y update && \
+ apt-get install -y openjdk-8-jdk && \
+ rm -rf /var/lib/apt/lists/*
+
+# Install Zeppelin
+ENV Z_VERSION="0.7.3" \
+ Z_HOME="/zeppelin"
+
+RUN echo "$LOG_TAG Download Zeppelin binary" && \
+ wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \
+ tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
+ rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
+ mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME}
+ENV PATH="${Z_HOME}/bin:${PATH}"
+
+RUN echo "$LOG_TAG Set locale" && \
+ echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
+ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \
+ echo "LANG=en_US.UTF-8" > /etc/locale.conf && \
+ locale-gen en_US.UTF-8
+
+ENV LANG=en_US.UTF-8 \
+ LC_ALL=en_US.UTF-8
+
+COPY zeppelin-site.xml $Z_HOME/conf/zeppelin-site.xml
+COPY shiro.ini ${Z_HOME}/conf/shiro.ini
+RUN chmod 777 -R ${Z_HOME}
+
+COPY run_container.sh /usr/local/bin/run_container.sh
+RUN chmod 755 /usr/local/bin/run_container.sh
+
+EXPOSE 8080
+CMD ["/usr/local/bin/run_container.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/run_container.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/run_container.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/run_container.sh
new file mode 100644
index 0000000..8b90920
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/run_container.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"${Z_HOME}/bin/zeppelin-daemon.sh" start
+while true; do
+ #perform the test
+ sleep 5
+done
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/shiro.ini
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/shiro.ini b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/shiro.ini
new file mode 100644
index 0000000..89f976a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/shiro.ini
@@ -0,0 +1,120 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[users]
+# List of users with their password allowed to access Zeppelin.
+# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
+# To enable admin user, uncomment the following line and set an appropriate password.
+admin = admin, admin
+user1 = password2, role1, role2
+user2 = password3, role3
+user3 = password4, role2
+
+# Sample LDAP configuration, for user Authentication, currently tested for single Realm
+[main]
+### A sample for configuring Active Directory Realm
+#activeDirectoryRealm = org.apache.zeppelin.realm.ActiveDirectoryGroupRealm
+#activeDirectoryRealm.systemUsername = userNameA
+
+#use either systemPassword or hadoopSecurityCredentialPath, more details in http://zeppelin.apache.org/docs/latest/security/shiroauthentication.html
+#activeDirectoryRealm.systemPassword = passwordA
+#activeDirectoryRealm.hadoopSecurityCredentialPath = jceks://file/user/zeppelin/zeppelin.jceks
+#activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
+#activeDirectoryRealm.url = ldap://ldap.test.com:389
+#activeDirectoryRealm.groupRolesMap = "CN=admin,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"admin","CN=finance,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"finance","CN=hr,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"hr"
+#activeDirectoryRealm.authorizationCachingEnabled = false
+
+### A sample for configuring LDAP Directory Realm
+#ldapRealm = org.apache.zeppelin.realm.LdapGroupRealm
+## search base for ldap groups (only relevant for LdapGroupRealm):
+#ldapRealm.contextFactory.environment[ldap.searchBase] = dc=COMPANY,dc=COM
+#ldapRealm.contextFactory.url = ldap://ldap.test.com:389
+#ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
+#ldapRealm.contextFactory.authenticationMechanism = simple
+
+### A sample PAM configuration
+#pamRealm=org.apache.zeppelin.realm.PamRealm
+#pamRealm.service=sshd
+
+### A sample for configuring ZeppelinHub Realm
+#zeppelinHubRealm = org.apache.zeppelin.realm.ZeppelinHubRealm
+## Url of ZeppelinHub
+#zeppelinHubRealm.zeppelinhubUrl = https://www.zeppelinhub.com
+#securityManager.realms = $zeppelinHubRealm
+
+## A same for configuring Knox SSO Realm
+#knoxJwtRealm = org.apache.zeppelin.realm.jwt.KnoxJwtRealm
+#knoxJwtRealm.providerUrl = https://domain.example.com/
+#knoxJwtRealm.login = gateway/knoxsso/knoxauth/login.html
+#knoxJwtRealm.logout = gateway/knoxssout/api/v1/webssout
+#knoxJwtRealm.logoutAPI = true
+#knoxJwtRealm.redirectParam = originalUrl
+#knoxJwtRealm.cookieName = hadoop-jwt
+#knoxJwtRealm.publicKeyPath = /etc/zeppelin/conf/knox-sso.pem
+#
+#knoxJwtRealm.groupPrincipalMapping = group.principal.mapping
+#knoxJwtRealm.principalMapping = principal.mapping
+#authc = org.apache.zeppelin.realm.jwt.KnoxAuthenticationFilter
+
+sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
+
+### If caching of user is required then uncomment below lines
+#cacheManager = org.apache.shiro.cache.MemoryConstrainedCacheManager
+#securityManager.cacheManager = $cacheManager
+
+### Enables 'HttpOnly' flag in Zeppelin cookies
+cookie = org.apache.shiro.web.servlet.SimpleCookie
+cookie.name = JSESSIONID
+cookie.httpOnly = true
+### Uncomment the below line only when Zeppelin is running over HTTPS
+#cookie.secure = true
+sessionManager.sessionIdCookie = $cookie
+
+securityManager.sessionManager = $sessionManager
+# 86,400,000 milliseconds = 24 hour
+securityManager.sessionManager.globalSessionTimeout = 86400000
+shiro.loginUrl = /api/login
+
+[roles]
+role1 = *
+role2 = *
+role3 = *
+admin = *
+
+[urls]
+# This section is used for url-based security. For details see the shiro.ini documentation.
+#
+# You can secure interpreter, configuration and credential information by urls.
+# Comment or uncomment the below urls that you want to hide:
+# anon means the access is anonymous.
+# authc means form based auth Security.
+#
+# IMPORTANT: Order matters: URL path expressions are evaluated against an incoming request
+# in the order they are defined and the FIRST MATCH WINS.
+#
+# To allow anonymous access to all but the stated urls,
+# uncomment the line second last line (/** = anon) and comment the last line (/** = authc)
+#
+/api/version = anon
+# Allow all authenticated users to restart interpreters on a notebook page.
+# Comment out the following line if you would like to authorize only admin users to restart interpreters.
+/api/interpreter/setting/restart/** = authc
+/api/interpreter/** = authc, roles[admin]
+/api/configurations/** = authc, roles[admin]
+/api/credential/** = authc, roles[admin]
+#/** = anon
+/** = authc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/zeppelin-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/zeppelin-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/zeppelin-site.xml
new file mode 100644
index 0000000..2bde161
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/docker/zeppelin-notebook-example/zeppelin-site.xml
@@ -0,0 +1,569 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+ <property>
+ <name>zeppelin.server.addr</name>
+ <value>0.0.0.0</value>
+ <description>Server address</description>
+ </property>
+
+ <property>
+ <name>zeppelin.server.port</name>
+ <value>8080</value>
+ <description>Server port.</description>
+ </property>
+
+ <property>
+ <name>zeppelin.server.ssl.port</name>
+ <value>8443</value>
+ <description>Server ssl port. (used when ssl property is set to true)</description>
+ </property>
+
+ <property>
+ <name>zeppelin.server.context.path</name>
+ <value>/</value>
+ <description>Context Path of the Web Application</description>
+ </property>
+
+ <property>
+ <name>zeppelin.war.tempdir</name>
+ <value>webapps</value>
+ <description>Location of jetty temporary directory</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.dir</name>
+ <value>notebook</value>
+ <description>path or URI for notebook persist</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.homescreen</name>
+ <value></value>
+ <description>id of notebook to be displayed in homescreen. ex) 2A94M5J1Z Empty value displays default home screen</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.homescreen.hide</name>
+ <value>false</value>
+ <description>hide homescreen notebook from list when this value set to true</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.collaborative.mode.enable</name>
+ <value>true</value>
+ <description>Enable collaborative mode</description>
+ </property>
+
+ <!-- Google Cloud Storage notebook storage -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.gcs.dir</name>
+ <value></value>
+ <description>
+ A GCS path in the form gs://bucketname/path/to/dir.
+ Notes are stored at {zeppelin.notebook.gcs.dir}/{notebook-id}/note.json
+ </description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.GCSNotebookRepo</value>
+ <description>notebook persistence layer implementation</description>
+ </property>
+ -->
+
+ <!-- Amazon S3 notebook storage -->
+ <!-- Creates the following directory structure: s3://{bucket}/{username}/{notebook-id}/note.json -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.s3.user</name>
+ <value>user</value>
+ <description>user name for s3 folder structure</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.s3.bucket</name>
+ <value>zeppelin</value>
+ <description>bucket name for notebook storage</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.s3.endpoint</name>
+ <value>s3.amazonaws.com</value>
+ <description>endpoint for s3 bucket</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.S3NotebookRepo</value>
+ <description>notebook persistence layer implementation</description>
+ </property>
+ -->
+
+ <!-- Additionally, encryption is supported for notebook data stored in S3 -->
+ <!-- Use the AWS KMS to encrypt data -->
+ <!-- If used, the EC2 role assigned to the EMR cluster must have rights to use the given key -->
+ <!-- See https://aws.amazon.com/kms/ and http://docs.aws.amazon.com/kms/latest/developerguide/concepts.html -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.s3.kmsKeyID</name>
+ <value>AWS-KMS-Key-UUID</value>
+ <description>AWS KMS key ID used to encrypt notebook data in S3</description>
+ </property>
+ -->
+
+ <!-- provide region of your KMS key -->
+ <!-- See http://docs.aws.amazon.com/general/latest/gr/rande.html#kms_region for region codes names -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.s3.kmsKeyRegion</name>
+ <value>us-east-1</value>
+ <description>AWS KMS key region in your AWS account</description>
+ </property>
+ -->
+
+ <!-- Use a custom encryption materials provider to encrypt data -->
+ <!-- No configuration is given to the provider, so you must use system properties or another means to configure -->
+ <!-- See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/EncryptionMaterialsProvider.html -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.s3.encryptionMaterialsProvider</name>
+ <value>provider implementation class name</value>
+ <description>Custom encryption materials provider used to encrypt notebook data in S3</description>
+ </property>
+ -->
+
+ <!-- Server-side encryption enabled for notebooks -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.s3.sse</name>
+ <value>true</value>
+ <description>Server-side encryption enabled for notebooks</description>
+ </property>
+ -->
+
+ <!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
+ <!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.s3.signerOverride</name>
+ <value>S3SignerType</value>
+ <description>optional override to control which signature algorithm should be used to sign AWS requests</description>
+ </property>
+ -->
+
+ <!-- If using Azure for storage use the following settings -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.azure.connectionString</name>
+ <value>DefaultEndpointsProtocol=https;AccountName=<accountName>;AccountKey=<accountKey></value>
+ <description>Azure account credentials</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.azure.share</name>
+ <value>zeppelin</value>
+ <description>share name for notebook storage</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.azure.user</name>
+ <value>user</value>
+ <description>optional user name for Azure folder structure</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.AzureNotebookRepo</value>
+ <description>notebook persistence layer implementation</description>
+ </property>
+ -->
+
+ <!-- Notebook storage layer using local file system
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
+ <description>local notebook persistence layer implementation</description>
+ </property>
+ -->
+
+ <!-- Notebook storage layer using hadoop compatible file system
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value>
+ <description>Hadoop compatible file system notebook persistence layer implementation, such as local file system, hdfs, azure wasb, s3 and etc.</description>
+ </property>
+
+ <property>
+ <name>zeppelin.server.kerberos.keytab</name>
+ <value></value>
+ <description>keytab for accessing kerberized hdfs</description>
+ </property>
+
+ <property>
+ <name>zeppelin.server.kerberos.principal</name>
+ <value></value>
+ <description>principal for accessing kerberized hdfs</description>
+ </property>
+ -->
+
+ <!-- For connecting your Zeppelin with ZeppelinHub -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.GitNotebookRepo, org.apache.zeppelin.notebook.repo.zeppelinhub.ZeppelinHubRepo</value>
+ <description>two notebook persistence layers (versioned local + ZeppelinHub)</description>
+ </property>
+ -->
+
+ <!-- MongoDB notebook storage -->
+ <!--
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.MongoNotebookRepo</value>
+ <description>notebook persistence layer implementation</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.mongo.uri</name>
+ <value>mongodb://localhost</value>
+ <description>MongoDB connection URI used to connect to a MongoDB database server</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.mongo.database</name>
+ <value>zeppelin</value>
+ <description>database name for notebook storage</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.mongo.collection</name>
+ <value>notes</value>
+ <description>collection name for notebook storage</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.mongo.autoimport</name>
+ <value>false</value>
+ <description>import local notes into MongoDB automatically on startup</description>
+ </property>
+ -->
+
+ <property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.GitNotebookRepo</value>
+ <description>versioned notebook persistence layer implementation</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.one.way.sync</name>
+ <value>false</value>
+ <description>If there are multiple notebook storages, should we treat the first one as the only source of truth?</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.dir</name>
+ <value>interpreter</value>
+ <description>Interpreter implementation base directory</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.localRepo</name>
+ <value>local-repo</value>
+ <description>Local repository for interpreter's additional dependency loading</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.dep.mvnRepo</name>
+ <value>http://repo1.maven.org/maven2/</value>
+ <description>Remote principal repository for interpreter's additional dependency loading</description>
+ </property>
+
+ <property>
+ <name>zeppelin.dep.localrepo</name>
+ <value>local-repo</value>
+ <description>Local repository for dependency loader</description>
+ </property>
+
+ <property>
+ <name>zeppelin.helium.node.installer.url</name>
+ <value>https://nodejs.org/dist/</value>
+ <description>Remote Node installer url for Helium dependency loader</description>
+ </property>
+
+ <property>
+ <name>zeppelin.helium.npm.installer.url</name>
+ <value>http://registry.npmjs.org/</value>
+ <description>Remote Npm installer url for Helium dependency loader</description>
+ </property>
+
+ <property>
+ <name>zeppelin.helium.yarnpkg.installer.url</name>
+ <value>https://github.com/yarnpkg/yarn/releases/download/</value>
+ <description>Remote Yarn package installer url for Helium dependency loader</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreters</name>
+ <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,
org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>
+ <description>Comma separated interpreter configurations. First interpreter become a default</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.group.order</name>
+ <value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam,groovy</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.connect.timeout</name>
+ <value>30000</value>
+ <description>Interpreter process connect timeout in msec.</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.output.limit</name>
+ <value>102400</value>
+ <description>Output message from interpreter exceeding the limit will be truncated</description>
+ </property>
+
+ <property>
+ <name>zeppelin.ssl</name>
+ <value>false</value>
+ <description>Should SSL be used by the servers?</description>
+ </property>
+
+ <property>
+ <name>zeppelin.ssl.client.auth</name>
+ <value>false</value>
+ <description>Should client authentication be used for SSL connections?</description>
+ </property>
+
+ <property>
+ <name>zeppelin.ssl.keystore.path</name>
+ <value>keystore</value>
+ <description>Path to keystore relative to Zeppelin configuration directory</description>
+ </property>
+
+ <property>
+ <name>zeppelin.ssl.keystore.type</name>
+ <value>JKS</value>
+ <description>The format of the given keystore (e.g. JKS or PKCS12)</description>
+ </property>
+
+ <property>
+ <name>zeppelin.ssl.keystore.password</name>
+ <value>change me</value>
+ <description>Keystore password. Can be obfuscated by the Jetty Password tool</description>
+ </property>
+
+ <!--
+ <property>
+ <name>zeppelin.ssl.key.manager.password</name>
+ <value>change me</value>
+ <description>Key Manager password. Defaults to keystore password. Can be obfuscated.</description>
+ </property>
+ -->
+
+ <property>
+ <name>zeppelin.ssl.truststore.path</name>
+ <value>truststore</value>
+ <description>Path to truststore relative to Zeppelin configuration directory. Defaults to the keystore path</description>
+ </property>
+
+ <property>
+ <name>zeppelin.ssl.truststore.type</name>
+ <value>JKS</value>
+ <description>The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type</description>
+ </property>
+
+ <!--
+ <property>
+ <name>zeppelin.ssl.truststore.password</name>
+ <value>change me</value>
+ <description>Truststore password. Can be obfuscated by the Jetty Password tool. Defaults to the keystore password</description>
+ </property>
+ -->
+
+ <property>
+ <name>zeppelin.server.allowed.origins</name>
+ <value>*</value>
+ <description>Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173</description>
+ </property>
+
+ <property>
+ <name>zeppelin.anonymous.allowed</name>
+ <value>false</value>
+ <description>Anonymous user allowed by default</description>
+ </property>
+
+ <property>
+ <name>zeppelin.username.force.lowercase</name>
+ <value>false</value>
+ <description>Force convert username case to lower case, useful for Active Directory/LDAP. Default is not to change case</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.default.owner.username</name>
+ <value></value>
+ <description>Set owner role by default</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.public</name>
+ <value>true</value>
+ <description>Make notebook public by default when created, private otherwise</description>
+ </property>
+
+ <property>
+ <name>zeppelin.websocket.max.text.message.size</name>
+ <value>1024000</value>
+ <description>Size in characters of the maximum text message to be received by websocket. Defaults to 1024000</description>
+ </property>
+
+ <property>
+ <name>zeppelin.server.default.dir.allowed</name>
+ <value>false</value>
+ <description>Enable directory listings on server.</description>
+ </property>
+
+ <!--
+ <property>
+ <name>zeppelin.interpreter.lifecyclemanager.class</name>
+ <value>org.apache.zeppelin.interpreter.lifecycle.TimeoutLifecycleManager</value>
+ <description>LifecycleManager class for managing the lifecycle of interpreters, by default interpreter will
+ be closed after timeout</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.lifecyclemanager.timeout.checkinterval</name>
+ <value>60000</value>
+ <description>Milliseconds of the interval to checking whether interpreter is time out</description>
+ </property>
+
+ <property>
+ <name>zeppelin.interpreter.lifecyclemanager.timeout.threshold</name>
+ <value>3600000</value>
+ <description>Milliseconds of the interpreter timeout threshold, by default it is 1 hour</description>
+ </property>
+ -->
+
+ <!--
+ <property>
+ <name>zeppelin.server.jetty.name</name>
+ <value>Jetty(7.6.0.v20120127)</value>
+ <description>Hardcoding Application Server name to Prevent Fingerprinting</description>
+ </property>
+ -->
+
+ <!--
+ <property>
+ <name>zeppelin.server.jetty.request.header.size</name>
+ <value>8192</value>
+ <description>Http Request Header Size Limit (to prevent HTTP 413)</description>
+ </property>
+ -->
+
+ <!--
+ <property>
+ <name>zeppelin.server.xframe.options</name>
+ <value>SAMEORIGIN</value>
+ <description>The X-Frame-Options HTTP response header can be used to indicate whether or not a browser should be allowed to render a page in a frame/iframe/object.</description>
+ </property>
+ -->
+
+ <!--
+ <property>
+ <name>zeppelin.server.strict.transport</name>
+ <value>max-age=631138519</value>
+ <description>The HTTP Strict-Transport-Security response header is a security feature that lets a web site tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. Enable this when Zeppelin is running on HTTPS. Value is in Seconds, the default value is equivalent to 20 years.</description>
+ </property>
+ -->
+ <!--
+
+ <property>
+ <name>zeppelin.server.xxss.protection</name>
+ <value>1</value>
+ <description>The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari that stops pages from loading when they detect reflected cross-site scripting (XSS) attacks. When value is set to 1 and a cross-site scripting attack is detected, the browser will sanitize the page (remove the unsafe parts).</description>
+ </property>
+ -->
+
+ <!--
+ <property>
+ <name>zeppelin.interpreter.callback.portRange</name>
+ <value>10000:10010</value>
+ </property>
+ -->
+
+ <!--
+ <property>
+ <name>zeppelin.recovery.storage.class</name>
+ <value>org.apache.zeppelin.interpreter.recovery.FileSystemRecoveryStorage</value>
+ <description>ReoveryStorage implementation</description>
+ </property>
+ -->
+
+ <!--
+ <property>
+ <name>zeppelin.recovery.dir</name>
+ <value>recovery</value>
+ <description>Location where recovery metadata is stored</description>
+ </property>
+ -->
+
+ <!-- GitHub configurations
+ <property>
+ <name>zeppelin.notebook.git.remote.url</name>
+ <value></value>
+ <description>remote Git repository URL</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.git.remote.username</name>
+ <value>token</value>
+ <description>remote Git repository username</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.git.remote.access-token</name>
+ <value></value>
+ <description>remote Git repository password</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.git.remote.origin</name>
+ <value>origin</value>
+ <description>Git repository remote</description>
+ </property>
+
+ <property>
+ <name>zeppelin.notebook.cron.enable</name>
+ <value>false</value>
+ <description>Notebook enable cron scheduler feature</description>
+ </property>
+ <property>
+ <name>zeppelin.notebook.cron.folders</name>
+ <value></value>
+ <description>Notebook cron folders</description>
+ </property>
+ -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/DeveloperGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/DeveloperGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/DeveloperGuide.md
deleted file mode 100644
index ce26ea7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/DeveloperGuide.md
+++ /dev/null
@@ -1,26 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Developper Guide
-
-(Need add more details)
-
-By default, submarine uses YARN service framework as runtime. If you want to add your own implementation. You can add a new `RuntimeFactory` implementation and configure following option to `submarine.xml` (which should be placed under same `$HADOOP_CONF_DIR`)
-
-```
-<property>
- <name>submarine.runtime.class</name>
- <value>... full qualified class name for your runtime factory ... </value>
-</property>
-```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/QuickStart.md
deleted file mode 100644
index b720b5a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/QuickStart.md
+++ /dev/null
@@ -1,134 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Quick Start Guide
-
-## Prerequisite
-
-Must:
-- Apache Hadoop 3.1.0, YARN service enabled.
-
-Optional:
-- Enable YARN DNS. (When distributed training required.)
-- Enable GPU on YARN support. (When GPU-based training required.)
-
-## Run jobs
-
-### Commandline options
-
-```$xslt
-usage: job run
- -checkpoint_path <arg> Training output directory of the job, could
- be local or other FS directory. This
- typically includes checkpoint files and
- exported model
- -docker_image <arg> Docker image name/tag
- -env <arg> Common environment variable of worker/ps
- -input_path <arg> Input of the job, could be local or other FS
- directory
- -name <arg> Name of the job
- -num_ps <arg> Number of PS tasks of the job, by default
- it's 0
- -num_workers <arg> Numnber of worker tasks of the job, by
- default it's 1
- -ps_docker_image <arg> Specify docker image for PS, when this is
- not specified, PS uses --docker_image as
- default.
- -ps_launch_cmd <arg> Commandline of worker, arguments will be
- directly used to launch the PS
- -ps_resources <arg> Resource of each PS, for example
- memory-mb=2048,vcores=2,yarn.io/gpu=2
- -queue <arg> Name of queue to run the job, by default it
- uses default queue
- -saved_model_path <arg> Model exported path (savedmodel) of the job,
- which is needed when exported model is not
- placed under ${checkpoint_path}could be
- local or other FS directory. This will be
- used to serve.
- -tensorboard <arg> Should we run TensorBoard for this job? By
- default it's true
- -verbose Print verbose log for troubleshooting
- -wait_job_finish Specified when user want to wait the job
- finish
- -worker_docker_image <arg> Specify docker image for WORKER, when this
- is not specified, WORKER uses --docker_image
- as default.
- -worker_launch_cmd <arg> Commandline of worker, arguments will be
- directly used to launch the worker
- -worker_resources <arg> Resource of each worker, for example
- memory-mb=2048,vcores=2,yarn.io/gpu=2
-```
-
-### Launch Standalone Tensorflow Application:
-
-#### Commandline
-```
-yarn jar path-to/hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar job run \
- --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \
- --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 --name tf-job-001 \
- --docker_image <your-docker-image> \
- --input_path hdfs://default/dataset/cifar-10-data \
- --checkpoint_path hdfs://default/tmp/cifar-10-jobdir \
- --worker_resources memory=4G,vcores=2,gpu=2 \
- --worker_launch_cmd "python ... (Your training application cmd)"
-```
-
-#### Notes:
-
-1) `DOCKER_JAVA_HOME` points to JAVA_HOME inside Docker image.
-2) `DOCKER_HADOOP_HDFS_HOME` points to HADOOP_HDFS_HOME inside Docker image.
-3) `--worker_resources` can include gpu when you need GPU to train your task.
-
-### Launch Distributed Tensorflow Application:
-
-#### Commandline
-
-```
-yarn jar hadoop-yarn-applications-submarine-<version>.jar job run \
- --name tf-job-001 --docker_image <your docker image> \
- --input_path hdfs://default/dataset/cifar-10-data \
- --checkpoint_path hdfs://default/tmp/cifar-10-jobdir \
- --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \
- --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 \
- --num_workers 2 \
- --worker_resources memory=8G,vcores=2,gpu=1 --worker_launch_cmd "cmd for worker ..." \
- --num_ps 2 \
- --ps_resources memory=4G,vcores=2,gpu=0 --ps_launch_cmd "cmd for ps" \
-```
-
-#### Notes:
-
-1) Very similar to standalone TF application, but you need to specify #worker/#ps
-2) Different resources can be specified for worker and PS.
-3) `TF_CONFIG` environment will be auto generated and set before executing user's launch command.
-
-## Run jobs
-
-### Get Job Status
-
-```
-yarn jar hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar job show --name tf-job-001
-```
-
-Output looks like:
-```
-Job Meta Info:
- Application Id: application_1532131617202_0005
- Input Path: hdfs://default/dataset/cifar-10-data
- Checkpoint Path: hdfs://default/tmp/cifar-10-jobdir
- Run Parameters: --name tf-job-001 --docker_image wtan/tf-1.8.0-gpu:0.0.3
- (... all your commandline before run the job)
-```
-
-After that, you can run ```tensorboard --logdir=<checkpoint-path>``` to view Tensorboard of the job.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/DeveloperGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/DeveloperGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/DeveloperGuide.md
new file mode 100644
index 0000000..76e3ae0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/DeveloperGuide.md
@@ -0,0 +1,24 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Developer Guide
+
+By default, submarine uses YARN service framework as runtime. If you want to add your own implementation. You can add a new `RuntimeFactory` implementation and configure following option to `submarine.xml` (which should be placed under same `$HADOOP_CONF_DIR`)
+
+```
+<property>
+ <name>submarine.runtime.class</name>
+ <value>... full qualified class name for your runtime factory ... </value>
+</property>
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Examples.md
new file mode 100644
index 0000000..3e7f02f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Examples.md
@@ -0,0 +1,21 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Examples
+
+Here're some examples about Submarine usage.
+
+[Running Distributed CIFAR 10 Tensorflow Job](RunningDistributedCifar10TFJobs.html)
+
+[Running Zeppelin Notebook on YARN](RunningZeppelinOnYARN.html)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md
new file mode 100644
index 0000000..0b78a87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md
@@ -0,0 +1,42 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Submarine is a project which allows infra engineer / data scientist to run *unmodified* Tensorflow programs on YARN.
+
+Goals of Submarine:
+
+- It allows jobs for easy access to data/models in HDFS and other storages.
+
+- Can launch services to serve Tensorflow/MXNet models.
+
+- Support run distributed Tensorflow jobs with simple configs.
+
+- Support run user-specified Docker images.
+
+- Support specify GPU and other resources.
+
+- Support launch tensorboard for training jobs if user specified.
+
+- Support customized DNS name for roles (like tensorboard.$user.$domain:6006)
+
+
+Click below contents if you want to understand more.
+
+- [QuickStart Guide](QuickStart.html)
+
+- [Examples](Examples.html)
+
+- [How to write Dockerfile for Submarine jobs](WriteDockerfile.html)
+
+- [Developer guide](DeveloperGuide.html)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md
new file mode 100644
index 0000000..da4fb95
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md
@@ -0,0 +1,174 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Quick Start Guide
+
+## Prerequisite
+
+Must:
+
+- Apache Hadoop 3.1.x, YARN service enabled.
+
+Optional:
+
+- Enable YARN DNS. (When distributed training is required.)
+- Enable GPU on YARN support. (When GPU-based training is required.)
+
+## Run jobs
+
+### Commandline options
+
+```$xslt
+usage: job run
+ -checkpoint_path <arg> Training output directory of the job, could
+ be local or other FS directory. This
+ typically includes checkpoint files and
+ exported model
+ -docker_image <arg> Docker image name/tag
+ -env <arg> Common environment variable of worker/ps
+ -input_path <arg> Input of the job, could be local or other FS
+ directory
+ -name <arg> Name of the job
+ -num_ps <arg> Number of PS tasks of the job, by default
+ it's 0
+ -num_workers <arg> Numnber of worker tasks of the job, by
+ default it's 1
+ -ps_docker_image <arg> Specify docker image for PS, when this is
+ not specified, PS uses --docker_image as
+ default.
+ -ps_launch_cmd <arg> Commandline of worker, arguments will be
+ directly used to launch the PS
+ -ps_resources <arg> Resource of each PS, for example
+ memory-mb=2048,vcores=2,yarn.io/gpu=2
+ -queue <arg> Name of queue to run the job, by default it
+ uses default queue
+ -saved_model_path <arg> Model exported path (savedmodel) of the job,
+ which is needed when exported model is not
+ placed under ${checkpoint_path}could be
+ local or other FS directory. This will be
+ used to serve.
+ -tensorboard <arg> Should we run TensorBoard for this job? By
+ default it's true
+ -verbose Print verbose log for troubleshooting
+ -wait_job_finish Specified when user want to wait the job
+ finish
+ -worker_docker_image <arg> Specify docker image for WORKER, when this
+ is not specified, WORKER uses --docker_image
+ as default.
+ -worker_launch_cmd <arg> Commandline of worker, arguments will be
+ directly used to launch the worker
+ -worker_resources <arg> Resource of each worker, for example
+ memory-mb=2048,vcores=2,yarn.io/gpu=2
+```
+
+### Launch Standalone Tensorflow Application:
+
+#### Commandline
+```
+yarn jar path-to/hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar job run \
+ --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \
+ --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 --name tf-job-001 \
+ --docker_image <your-docker-image> \
+ --input_path hdfs://default/dataset/cifar-10-data \
+ --checkpoint_path hdfs://default/tmp/cifar-10-jobdir \
+ --worker_resources memory=4G,vcores=2,gpu=2 \
+ --worker_launch_cmd "python ... (Your training application cmd)" \
+ --tensorboard # this will launch a companion tensorboard container for monitoring
+```
+
+#### Notes:
+
+1) `DOCKER_JAVA_HOME` points to JAVA_HOME inside Docker image.
+
+2) `DOCKER_HADOOP_HDFS_HOME` points to HADOOP_HDFS_HOME inside Docker image.
+
+3) `--worker_resources` can include gpu when you need GPU to train your task.
+
+4) When `--tensorboard` is specified, you can go to YARN new UI, go to services -> `<you specified service>` -> Click `...` to access Tensorboard.
+
+This will launch a Tensorboard to monitor *all your jobs*. By access YARN UI (the new UI). You can go to services page, go to the `tensorboard-service`, click quick links (`Tensorboard`) can lead you to the tensorboard.
+
+See below screenshot:
+
+
+
+### Launch Distributed Tensorflow Application:
+
+#### Commandline
+
+```
+yarn jar hadoop-yarn-applications-submarine-<version>.jar job run \
+ --name tf-job-001 --docker_image <your docker image> \
+ --input_path hdfs://default/dataset/cifar-10-data \
+ --checkpoint_path hdfs://default/tmp/cifar-10-jobdir \
+ --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \
+ --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 \
+ --num_workers 2 \
+ --worker_resources memory=8G,vcores=2,gpu=1 --worker_launch_cmd "cmd for worker ..." \
+ --num_ps 2 \
+ --ps_resources memory=4G,vcores=2,gpu=0 --ps_launch_cmd "cmd for ps" \
+```
+
+#### Notes:
+
+1) Very similar to standalone TF application, but you need to specify #worker/#ps
+
+2) Different resources can be specified for worker and PS.
+
+3) `TF_CONFIG` environment will be auto generated and set before executing user's launch command.
+
+## Get job history / logs
+
+### Get Job Status from CLI
+
+```
+yarn jar hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar job show --name tf-job-001
+```
+
+Output looks like:
+```
+Job Meta Info:
+ Application Id: application_1532131617202_0005
+ Input Path: hdfs://default/dataset/cifar-10-data
+ Checkpoint Path: hdfs://default/tmp/cifar-10-jobdir
+ Run Parameters: --name tf-job-001 --docker_image wtan/tf-1.8.0-gpu:0.0.3
+ (... all your commandline before run the job)
+```
+
+After that, you can run ```tensorboard --logdir=<checkpoint-path>``` to view Tensorboard of the job.
+
+### Run tensorboard to monitor your jobs
+
+```
+# Cleanup previous service if needed
+yarn app -destroy tensorboard-service; \
+yarn jar /tmp/hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar \
+ job run --name tensorboard-service --verbose --docker_image wtan/tf-1.8.0-cpu:0.0.3 \
+ --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/ \
+ --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0 \
+ --num_workers 0 --tensorboard
+```
+
+You can view multiple job training history like from the `Tensorboard` link:
+
+
+
+
+### Get component logs from a training job
+
+There're two ways to get training job logs, one is from YARN UI (new or old):
+
+
+
+Or you can use `yarn logs -applicationId <applicationId>` to get logs from CLI
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningDistributedCifar10TFJobs.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningDistributedCifar10TFJobs.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningDistributedCifar10TFJobs.md
new file mode 100644
index 0000000..127c80f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningDistributedCifar10TFJobs.md
@@ -0,0 +1,162 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+# Tutorial: Running Distributed Cifar10 Tensorflow Estimator Example.
+
+## Prepare data for training
+
+CIFAR-10 is a common benchmark in machine learning for image recognition. Below example is based on CIFAR-10 dataset.
+
+1) Checkout https://github.com/tensorflow/models/:
+```
+git clone https://github.com/tensorflow/models/
+```
+
+2) Go to `models/tutorials/image/cifar10_estimator`
+
+3) Generate data by using following command: (required Tensorflow installed)
+
+```
+python generate_cifar10_tfrecords.py --data-dir=cifar-10-data
+```
+
+4) Upload data to HDFS
+
+```
+hadoop fs -put cifar-10-data/ /dataset/cifar-10-data
+```
+
+**Please note that:**
+
+YARN service doesn't allow multiple services with the same name, so please run following command
+```
+yarn application -destroy <service-name>
+```
+to delete services if you want to reuse the same service name.
+
+## Prepare Docker images
+
+Refer to [Write Dockerfile](WriteDockerfile.md) to build a Docker image or use prebuilt one.
+
+## Run Tensorflow jobs
+
+### Run standalone training
+
+```
+yarn jar path/to/hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar \
+ job run --name tf-job-001 --verbose --docker_image hadoopsubmarine/tf-1.8.0-gpu:0.0.1 \
+ --input_path hdfs://default/dataset/cifar-10-data \
+ --env DOCKER_JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre/
+ --env DOCKER_HADOOP_HDFS_HOME=/hadoop-3.1.0
+ --num_workers 1 --worker_resources memory=8G,vcores=2,gpu=1 \
+ --worker_launch_cmd "cd /test/models/tutorials/image/cifar10_estimator && python cifar10_main.py --data-dir=%input_path% --job-dir=%checkpoint_path% --train-steps=10000 --eval-batch-size=16 --train-batch-size=16 --num-gpus=2 --sync" \
+ --tensorboard --tensorboard_docker_image wtan/tf-1.8.0-cpu:0.0.3
+```
+
+Explanations:
+
+- When access of HDFS is required, the two environments are required to indicate: JAVA_HOME and HDFS_HOME to access libhdfs libraries *inside Docker image*. We will try to eliminate specifying this in the future.
+- Docker image for worker and tensorboard can be specified separately. For this case, Tensorboard doesn't need GPU, so we will use cpu Docker image for Tensorboard. (Same for parameter-server in the distributed example below).
+
+### Run distributed training
+
+```
+yarn jar path/to/hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar \
+ job run --name tf-job-001 --verbose --docker_image hadoopsubmarine/tf-1.8.0-gpu:0.0.1 \
+ --input_path hdfs://default/dataset/cifar-10-data \
+ --env(s) (same as standalone)
+ --num_workers 2 \
+ --worker_resources memory=8G,vcores=2,gpu=1 \
+ --worker_launch_cmd "cd /test/models/tutorials/image/cifar10_estimator && python cifar10_main.py --data-dir=%input_path% --job-dir=%checkpoint_path% --train-steps=10000 --eval-batch-size=16 --train-batch-size=16 --num-gpus=2 --sync" \
+ --ps_docker_image wtan/tf-1.8.0-cpu:0.0.3 \
+ --num_ps 1 --ps_resources memory=4G,vcores=2,gpu=0 \
+ --ps_launch_cmd "cd /test/models/tutorials/image/cifar10_estimator && python cifar10_main.py --data-dir=%input_path% --job-dir=%checkpoint_path% --num-gpus=0" \
+ --tensorboard --tensorboard_docker_image wtan/tf-1.8.0-cpu:0.0.3
+```
+
+Explanations:
+
+- `>1` num_workers indicates it is a distributed training.
+- Parameters / resources / Docker image of parameter server can be specified separately. For many cases, parameter server doesn't require GPU.
+
+*Outputs of distributed training*
+
+Sample output of master:
+```
+...
+allow_soft_placement: true
+, '_tf_random_seed': None, '_task_type': u'master', '_environment': u'cloud', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fe77cb15050>, '_tf_config': gpu_options {
+ per_process_gpu_memory_fraction: 1.0
+}
+...
+2018-05-06 22:29:14.656022: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job master -> {0 -> localhost:8000}
+2018-05-06 22:29:14.656097: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> ps-0.distributed-tf.root.tensorflow.site:8000}
+2018-05-06 22:29:14.656112: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> worker-0.distributed-tf.root.tensorflow.site:8000}
+2018-05-06 22:29:14.659359: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000
+...
+INFO:tensorflow:Restoring parameters from hdfs://default/tmp/cifar-10-jobdir/model.ckpt-0
+INFO:tensorflow:Evaluation [1/625]
+INFO:tensorflow:Evaluation [2/625]
+INFO:tensorflow:Evaluation [3/625]
+INFO:tensorflow:Evaluation [4/625]
+INFO:tensorflow:Evaluation [5/625]
+INFO:tensorflow:Evaluation [6/625]
+...
+INFO:tensorflow:Validation (step 1): loss = 1220.6445, global_step = 1, accuracy = 0.1
+INFO:tensorflow:loss = 6.3980675, step = 0
+INFO:tensorflow:loss = 6.3980675, learning_rate = 0.1
+INFO:tensorflow:global_step/sec: 2.34092
+INFO:tensorflow:Average examples/sec: 1931.22 (1931.22), step = 100
+INFO:tensorflow:Average examples/sec: 354.236 (38.6479), step = 110
+INFO:tensorflow:Average examples/sec: 211.096 (38.7693), step = 120
+INFO:tensorflow:Average examples/sec: 156.533 (38.1633), step = 130
+INFO:tensorflow:Average examples/sec: 128.6 (38.7372), step = 140
+INFO:tensorflow:Average examples/sec: 111.533 (39.0239), step = 150
+```
+
+Sample output of worker:
+```
+, '_tf_random_seed': None, '_task_type': u'worker', '_environment': u'cloud', '_is_chief': False, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fc2a490b050>, '_tf_config': gpu_options {
+ per_process_gpu_memory_fraction: 1.0
+}
+...
+2018-05-06 22:28:45.807936: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job master -> {0 -> master-0.distributed-tf.root.tensorflow.site:8000}
+2018-05-06 22:28:45.808040: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> ps-0.distributed-tf.root.tensorflow.site:8000}
+2018-05-06 22:28:45.808064: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> localhost:8000}
+2018-05-06 22:28:45.809919: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000
+...
+INFO:tensorflow:loss = 5.319096, step = 0
+INFO:tensorflow:loss = 5.319096, learning_rate = 0.1
+INFO:tensorflow:Average examples/sec: 49.2338 (49.2338), step = 10
+INFO:tensorflow:Average examples/sec: 52.117 (55.3589), step = 20
+INFO:tensorflow:Average examples/sec: 53.2754 (55.7541), step = 30
+INFO:tensorflow:Average examples/sec: 53.8388 (55.6028), step = 40
+INFO:tensorflow:Average examples/sec: 54.1082 (55.2134), step = 50
+INFO:tensorflow:Average examples/sec: 54.3141 (55.3676), step = 60
+```
+
+Sample output of ps:
+```
+...
+, '_tf_random_seed': None, '_task_type': u'ps', '_environment': u'cloud', '_is_chief': False, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4be54dff90>, '_tf_config': gpu_options {
+ per_process_gpu_memory_fraction: 1.0
+}
+...
+2018-05-06 22:28:42.562316: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job master -> {0 -> master-0.distributed-tf.root.tensorflow.site:8000}
+2018-05-06 22:28:42.562408: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:8000}
+2018-05-06 22:28:42.562433: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> worker-0.distributed-tf.root.tensorflow.site:8000}
+2018-05-06 22:28:42.564242: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:316] Started server with target: grpc://localhost:8000
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningZeppelinOnYARN.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningZeppelinOnYARN.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningZeppelinOnYARN.md
new file mode 100644
index 0000000..e06526c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/RunningZeppelinOnYARN.md
@@ -0,0 +1,37 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Running Zeppelin Notebook On Submarine
+
+This is a simple example about how to run Zeppelin notebook by using Submarine.
+
+## Step 1: Build Docker Image
+
+Go to `src/main/docker/zeppelin-notebook-example`, build the Docker image. Or you can use the prebuilt one: `hadoopsubmarine/zeppelin-on-yarn-gpu:0.0.1`
+
+## Step 2: Launch the notebook on YARN
+
+Submit command to YARN:
+
+`yarn app -destroy zeppelin-notebook;
+yarn jar path-to/hadoop-yarn-applications-submarine-3.2.0-SNAPSHOT.jar \
+ job run --name zeppelin-notebook \
+ --docker_image hadoopsubmarine/zeppelin-on-yarn-gpu:0.0.1 \
+ --worker_resources memory=8G,vcores=2,gpu=1 \
+ --num_workers 1 \
+ -worker_launch_cmd "/usr/local/bin/run_container.sh"`
+
+Once the container got launched, you can go to `YARN services` UI page, access the `zeppelin-notebook` job, and go to the quicklink `notebook` by clicking `...`.
+
+The notebook is secured by admin/admin user name and password.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/WriteDockerfile.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/WriteDockerfile.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/WriteDockerfile.md
new file mode 100644
index 0000000..79aac8d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/WriteDockerfile.md
@@ -0,0 +1,117 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Creating Docker Images for Running Tensorflow on YARN
+
+## How to create docker images to run Tensorflow on YARN
+
+Dockerfile to run Tensorflow on YARN need two part:
+
+**Base libraries which Tensorflow depends on**
+
+1) OS base image, for example ```ubuntu:16.04```
+
+2) Tensorflow depended libraries and packages. For example ```python```, ```scipy```. For GPU support, need ```cuda```, ```cudnn```, etc.
+
+3) Tensorflow package.
+
+**Libraries to access HDFS**
+
+1) JDK
+
+2) Hadoop
+
+Here's an example of a base image (w/o GPU support) to install Tensorflow:
+```
+FROM ubuntu:16.04
+
+# Pick up some TF dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ build-essential \
+ curl \
+ libfreetype6-dev \
+ libpng12-dev \
+ libzmq3-dev \
+ pkg-config \
+ python \
+ python-dev \
+ rsync \
+ software-properties-common \
+ unzip \
+ && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py && \
+ rm get-pip.py
+
+RUN pip --no-cache-dir install \
+ Pillow \
+ h5py \
+ ipykernel \
+ jupyter \
+ matplotlib \
+ numpy \
+ pandas \
+ scipy \
+ sklearn \
+ && \
+ python -m ipykernel.kernelspec
+
+RUN pip --no-cache-dir install \
+ http://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.8.0-cp27-none-linux_x86_64.whl
+```
+
+On top of above image, add files, install packages to access HDFS
+```
+RUN apt-get update && apt-get install -y openjdk-8-jdk wget
+RUN wget http://apache.cs.utah.edu/hadoop/common/hadoop-3.1.0/hadoop-3.1.0.tar.gz
+RUN tar zxf hadoop-3.1.0.tar.gz
+```
+
+Build and push to your own docker registry: Use ```docker build ... ``` and ```docker push ...``` to finish this step.
+
+## Use examples to build your own Tensorflow docker images
+
+We provided following examples for you to build tensorflow docker images.
+
+For Tensorflow 1.8.0 (Precompiled to CUDA 9.x)
+
+- *docker/base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0*: Tensorflow 1.8.0 supports CPU only.
+- *docker/with-cifar10-models/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0*: Tensorflow 1.8.0 supports CPU only, and included models
+- *docker/base/ubuntu-16.04/Dockerfile.gpu.cuda_9.0.tf_1.8.0*: Tensorflow 1.8.0 supports GPU, which is prebuilt to CUDA9.
+- *docker/with-cifar10-models/ubuntu-16.04/Dockerfile.gpu.cuda_8.0.tf_1.8.0*: Tensorflow 1.8.0 supports GPU, which is prebuilt to CUDA9, with models.
+
+## Build Docker images
+
+### Manually build Docker image:
+
+Under `docker/` directory, run `build-all.sh` to build Docker images. It will build following images:
+
+- `tf-1.8.0-gpu-base:0.0.1` for base Docker image which includes Hadoop, Tensorflow, GPU base libraries.
+- `tf-1.8.0-gpu-base:0.0.1` for base Docker image which includes Hadoop. Tensorflow.
+- `tf-1.8.0-gpu:0.0.1` which includes cifar10 model
+- `tf-1.8.0-cpu:0.0.1` which inclues cifar10 model (cpu only).
+
+### Use prebuilt images
+
+(No liability)
+You can also use prebuilt images for convenience:
+
+- hadoopsubmarine/tf-1.8.0-gpu:0.0.1
+- hadoopsubmarine/tf-1.8.0-cpu:0.0.1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/css/site.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/css/site.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/css/site.css
new file mode 100644
index 0000000..7315db3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/css/site.css
@@ -0,0 +1,29 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+ height: 93px;
+ background: none;
+}
+
+#bannerLeft img {
+ margin-left: 30px;
+ margin-top: 10px;
+}
+
+#bannerRight img {
+ margin: 17px;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/job-logs-ui.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/job-logs-ui.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/job-logs-ui.png
new file mode 100644
index 0000000..18b9e17
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/job-logs-ui.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/multiple-tensorboard-jobs.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/multiple-tensorboard-jobs.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/multiple-tensorboard-jobs.png
new file mode 100644
index 0000000..8e3db79
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/multiple-tensorboard-jobs.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/tensorboard-service.png
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/tensorboard-service.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/tensorboard-service.png
new file mode 100644
index 0000000..3251d74
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/resources/images/tensorboard-service.png differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19ad5be6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/site.xml
new file mode 100644
index 0000000..5feae9a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/site.xml
@@ -0,0 +1,28 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project name="Apache Hadoop ${project.version}">
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-stylus-skin</artifactId>
+ <version>${maven-stylus-skin.version}</version>
+ </skin>
+
+ <body>
+ <links>
+ <item name="Apache Hadoop" href="http://hadoop.apache.org/"/>
+ </links>
+ </body>
+
+</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-8468. Enable the use of queue based maximum container allocation limit and implement it in FairScheduler. Contributed by Antal Bálint Steinbach.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
index 078b8fd..c80469b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
@@ -18,26 +18,108 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.apache.hadoop.yarn.api.records.ContainerUpdateType.INCREASE_RESOURCE;
+import static org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils.RESOURCE_OUTSIDE_ALLOWED_RANGE;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerError;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.impl.pb.UpdateContainerRequestPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
public class TestRMServerUtils {
+
+ @Test
+ public void testValidateAndSplitUpdateResourceRequests() {
+ List<UpdateContainerRequest> updateRequests = new ArrayList<>();
+ int containerVersion = 10;
+ int resource = 10;
+ Resource maxAllocation = Resource.newInstance(resource, resource);
+
+ UpdateContainerRequestPBImpl updateContainerRequestPBFail =
+ new UpdateContainerRequestPBImpl();
+ updateContainerRequestPBFail.setContainerVersion(containerVersion);
+ updateContainerRequestPBFail
+ .setCapability(Resource.newInstance(resource + 1, resource + 1));
+ updateContainerRequestPBFail
+ .setContainerId(Mockito.mock(ContainerId.class));
+
+ ContainerId containerIdOk = Mockito.mock(ContainerId.class);
+ Resource capabilityOk = Resource.newInstance(resource - 1, resource - 1);
+ UpdateContainerRequestPBImpl updateContainerRequestPBOk =
+ new UpdateContainerRequestPBImpl();
+ updateContainerRequestPBOk.setContainerVersion(containerVersion);
+ updateContainerRequestPBOk.setCapability(capabilityOk);
+ updateContainerRequestPBOk.setContainerUpdateType(INCREASE_RESOURCE);
+ updateContainerRequestPBOk.setContainerId(containerIdOk);
+
+ updateRequests.add(updateContainerRequestPBOk);
+ updateRequests.add(updateContainerRequestPBFail);
+
+ Dispatcher dispatcher = Mockito.mock(Dispatcher.class);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+
+ Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+ Mockito.when(rmContext.getDispatcher()).thenReturn(dispatcher);
+
+ RMContainer rmContainer = Mockito.mock(RMContainer.class);
+ Mockito.when(scheduler.getRMContainer(Mockito.any()))
+ .thenReturn(rmContainer);
+ Container container = Mockito.mock(Container.class);
+ Mockito.when(container.getVersion()).thenReturn(containerVersion);
+ Mockito.when(rmContainer.getContainer()).thenReturn(container);
+ Mockito.when(scheduler.getNormalizedResource(capabilityOk, maxAllocation))
+ .thenReturn(capabilityOk);
+
+ AllocateRequest allocateRequest =
+ AllocateRequest.newInstance(1, 0.5f, new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>(), updateRequests, null);
+
+ List<UpdateContainerError> updateErrors = new ArrayList<>();
+ ContainerUpdates containerUpdates =
+ RMServerUtils.validateAndSplitUpdateResourceRequests(rmContext,
+ allocateRequest, maxAllocation, updateErrors);
+ Assert.assertEquals(1, updateErrors.size());
+ Assert.assertEquals(resource + 1, updateErrors.get(0)
+ .getUpdateContainerRequest().getCapability().getMemorySize());
+ Assert.assertEquals(resource + 1, updateErrors.get(0)
+ .getUpdateContainerRequest().getCapability().getVirtualCores());
+ Assert.assertEquals(RESOURCE_OUTSIDE_ALLOWED_RANGE,
+ updateErrors.get(0).getReason());
+
+ Assert.assertEquals(1, containerUpdates.getIncreaseRequests().size());
+ UpdateContainerRequest increaseRequest =
+ containerUpdates.getIncreaseRequests().get(0);
+ Assert.assertEquals(capabilityOk.getVirtualCores(),
+ increaseRequest.getCapability().getVirtualCores());
+ Assert.assertEquals(capabilityOk.getMemorySize(),
+ increaseRequest.getCapability().getMemorySize());
+ Assert.assertEquals(containerIdOk, increaseRequest.getContainerId());
+ }
+
@Test
public void testGetApplicableNodeCountForAMLocality() throws Exception {
List<NodeId> rack1Nodes = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 2ec2de2..6969242 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -106,6 +106,7 @@ import org.junit.Test;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.junit.rules.ExpectedException;
+import org.mockito.Mockito;
public class TestSchedulerUtils {
@@ -271,7 +272,7 @@ public class TestSchedulerUtils {
public void testValidateResourceRequestWithErrorLabelsPermission()
throws IOException {
// mock queue and scheduler
- YarnScheduler scheduler = mock(YarnScheduler.class);
+ ResourceScheduler scheduler = mock(ResourceScheduler.class);
Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
QueueInfo queueInfo = mock(QueueInfo.class);
when(queueInfo.getQueueName()).thenReturn("queue");
@@ -280,6 +281,8 @@ public class TestSchedulerUtils {
when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean()))
.thenReturn(queueInfo);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+
Resource maxResource = Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
@@ -298,20 +301,20 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
resReq.setNodeLabelExpression("y");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
resReq.setNodeLabelExpression("");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
resReq.setNodeLabelExpression(" ");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is a subset of queue labels");
@@ -332,8 +335,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
@@ -354,8 +357,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("z");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
} finally {
@@ -379,8 +382,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x && y");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
} finally {
@@ -399,16 +402,16 @@ public class TestSchedulerUtils {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
resReq.setNodeLabelExpression("");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
resReq.setNodeLabelExpression(" ");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is empty");
@@ -428,8 +431,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidLabelResourceRequestException e) {
invalidlabelexception = true;
@@ -456,16 +459,16 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
resReq.setNodeLabelExpression("y");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
resReq.setNodeLabelExpression("z");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when queue can access any labels");
@@ -486,8 +489,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
}
@@ -507,8 +510,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
} finally {
@@ -532,8 +535,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
} finally {
@@ -545,8 +548,8 @@ public class TestSchedulerUtils {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
ResourceRequest resReq1 = BuilderUtils
.newResourceRequest(mock(Priority.class), "*", resource, 1, "x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq1, "queue",
+ scheduler, rmContext, maxResource);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
assertEquals("Invalid label resource request, cluster do not contain , "
@@ -560,8 +563,8 @@ public class TestSchedulerUtils {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
ResourceRequest resReq1 = BuilderUtils
.newResourceRequest(mock(Priority.class), "*", resource, 1, "x");
- SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq1, "queue",
+ scheduler, rmContext, maxResource);
Assert.assertEquals(RMNodeLabelsManager.NO_LABEL,
resReq1.getNodeLabelExpression());
} catch (InvalidResourceRequestException e) {
@@ -571,14 +574,21 @@ public class TestSchedulerUtils {
}
@Test(timeout = 30000)
- public void testValidateResourceRequest() {
- YarnScheduler mockScheduler = mock(YarnScheduler.class);
+ public void testValidateResourceRequest() throws IOException {
+ ResourceScheduler mockScheduler = mock(ResourceScheduler.class);
+
+ QueueInfo queueInfo = mock(QueueInfo.class);
+ when(queueInfo.getQueueName()).thenReturn("queue");
Resource maxResource =
Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ when(rmContext.getScheduler()).thenReturn(mockScheduler);
+ when(mockScheduler.getQueueInfo(Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean())).thenReturn(queueInfo);
+
// zero memory
try {
Resource resource =
@@ -587,8 +597,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
} catch (InvalidResourceRequestException e) {
fail("Zero memory should be accepted");
}
@@ -601,8 +611,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
} catch (InvalidResourceRequestException e) {
fail("Zero vcores should be accepted");
}
@@ -616,8 +626,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
} catch (InvalidResourceRequestException e) {
fail("Max memory should be accepted");
}
@@ -631,8 +641,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
} catch (InvalidResourceRequestException e) {
fail("Max vcores should not be accepted");
}
@@ -645,8 +655,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
fail("Negative memory should not be accepted");
} catch (InvalidResourceRequestException e) {
assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType());
@@ -660,8 +670,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
fail("Negative vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType());
@@ -676,8 +686,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
fail("More than max memory should not be accepted");
} catch (InvalidResourceRequestException e) {
assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType());
@@ -691,8 +701,8 @@ public class TestSchedulerUtils {
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, null,
+ mockScheduler, rmContext, maxResource);
fail("More than max vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType());
@@ -805,7 +815,7 @@ public class TestSchedulerUtils {
public void testNormalizeNodeLabelExpression()
throws IOException {
// mock queue and scheduler
- YarnScheduler scheduler = mock(YarnScheduler.class);
+ ResourceScheduler scheduler = mock(ResourceScheduler.class);
Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
QueueInfo queueInfo = mock(QueueInfo.class);
when(queueInfo.getQueueName()).thenReturn("queue");
@@ -818,6 +828,8 @@ public class TestSchedulerUtils {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+
// queue has labels, success cases
try {
// set queue accessible node labels to [x, y]
@@ -831,13 +843,13 @@ public class TestSchedulerUtils {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
Assert.assertEquals("x", resReq.getNodeLabelExpression());
resReq.setNodeLabelExpression(" y ");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler, rmContext);
+ normalizeAndvalidateRequest(resReq, "queue",
+ scheduler, rmContext, maxResource);
Assert.assertEquals("y", resReq.getNodeLabelExpression());
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
@@ -1033,6 +1045,14 @@ public class TestSchedulerUtils {
return rmContext;
}
+ private static void normalizeAndvalidateRequest(ResourceRequest resReq,
+ String queueName, YarnScheduler scheduler, RMContext rmContext,
+ Resource maxAllocation)
+ throws InvalidResourceRequestException {
+ SchedulerUtils.normalizeAndValidateRequest(resReq, maxAllocation, queueName,
+ scheduler, rmContext, null);
+ }
+
private static class InvalidResourceRequestExceptionMessageGenerator {
private StringBuilder sb;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index e77d8e2..f32b467 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -835,6 +837,41 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
}
+ @Test
+ public void testQueueMaximumAllocations() {
+ CapacityScheduler scheduler = new CapacityScheduler();
+ scheduler.setConf(new YarnConfiguration());
+ scheduler.setRMContext(resourceManager.getRMContext());
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+
+ setupQueueConfiguration(conf);
+ conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1)
+ + MAXIMUM_ALLOCATION_MB, "1024");
+ conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1)
+ + MAXIMUM_ALLOCATION_VCORES, "1");
+
+ scheduler.init(conf);
+ scheduler.start();
+
+ Resource maxAllocationForQueue =
+ scheduler.getMaximumResourceCapability("a1");
+ Resource maxAllocation1 = scheduler.getMaximumResourceCapability("");
+ Resource maxAllocation2 = scheduler.getMaximumResourceCapability(null);
+ Resource maxAllocation3 = scheduler.getMaximumResourceCapability();
+
+ Assert.assertEquals(maxAllocation1, maxAllocation2);
+ Assert.assertEquals(maxAllocation1, maxAllocation3);
+ Assert.assertEquals(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ maxAllocation1.getMemorySize());
+ Assert.assertEquals(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ maxAllocation1.getVirtualCores());
+
+ Assert.assertEquals(1024, maxAllocationForQueue.getMemorySize());
+ Assert.assertEquals(1, maxAllocationForQueue.getVirtualCores());
+ }
+
@Test
public void testRefreshQueues() throws Exception {
@@ -4009,7 +4046,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
private void setMaxAllocMb(CapacitySchedulerConfiguration conf,
String queueName, int maxAllocMb) {
String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
- + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
+ + MAXIMUM_ALLOCATION_MB;
conf.setInt(propName, maxAllocMb);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 3ac3849..4f1f20b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -77,6 +77,7 @@ public class FairSchedulerTestBase {
public static final float TEST_RESERVATION_THRESHOLD = 0.09f;
private static final int SLEEP_DURATION = 10;
private static final int SLEEP_RETRIES = 1000;
+ protected static final int RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE = 10240;
final static ContainerUpdates NULL_UPDATE_REQUESTS =
new ContainerUpdates();
@@ -93,7 +94,8 @@ public class FairSchedulerTestBase {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
1024);
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE);
conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false);
conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10);
conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 50a003e..ac30b23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue;
@@ -30,7 +31,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
@@ -41,6 +44,7 @@ import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -53,6 +57,8 @@ import static org.junit.Assert.fail;
public class TestAllocationFileLoaderService {
+ private static final String A_CUSTOM_RESOURCE = "a-custom-resource";
+
final static String TEST_DIR = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
@@ -203,7 +209,8 @@ public class TestAllocationFileLoaderService {
@Test
public void testAllocationFileParsing() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = new YarnConfiguration();
+ TestResourceUtils.addNewTypesToResources(A_CUSTOM_RESOURCE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
@@ -246,6 +253,8 @@ public class TestAllocationFileLoaderService {
.fairSharePreemptionTimeout(120)
.minSharePreemptionTimeout(50)
.fairSharePreemptionThreshold(0.6)
+ .maxContainerAllocation(
+ "vcores=16, memory-mb=512, " + A_CUSTOM_RESOURCE + "=10")
// Create hierarchical queues G,H, with different min/fair
// share preemption timeouts and preemption thresholds.
// Also add a child default to make sure it doesn't impact queue H.
@@ -253,6 +262,7 @@ public class TestAllocationFileLoaderService {
.fairSharePreemptionTimeout(180)
.minSharePreemptionTimeout(40)
.fairSharePreemptionThreshold(0.7)
+ .maxContainerAllocation("1024mb,8vcores")
.buildSubQueue()
.buildQueue()
// Set default limit of apps per queue to 15
@@ -286,8 +296,6 @@ public class TestAllocationFileLoaderService {
assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
- assertEquals(Resources.createResource(0),
- queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(2048, 10),
queueConf.getMaxResources("root.queueA").getResource());
@@ -358,6 +366,29 @@ public class TestAllocationFileLoaderService {
assertEquals(.4f, queueConf.getQueueMaxAMShare("root.queueD"), 0.01);
assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueE"), 0.01);
+ Resource expectedResourceWithCustomType = Resources.createResource(512, 16);
+ expectedResourceWithCustomType.setResourceValue(A_CUSTOM_RESOURCE, 10);
+
+ assertEquals(Resources.unbounded(),
+ queueConf.getQueueMaxContainerAllocation(
+ "root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
+ assertEquals(Resources.unbounded(),
+ queueConf.getQueueMaxContainerAllocation("root.queueA"));
+ assertEquals(Resources.unbounded(),
+ queueConf.getQueueMaxContainerAllocation("root.queueB"));
+ assertEquals(Resources.unbounded(),
+ queueConf.getQueueMaxContainerAllocation("root.queueC"));
+ assertEquals(Resources.unbounded(),
+ queueConf.getQueueMaxContainerAllocation("root.queueD"));
+ assertEquals(Resources.unbounded(),
+ queueConf.getQueueMaxContainerAllocation("root.queueE"));
+ assertEquals(Resources.unbounded(),
+ queueConf.getQueueMaxContainerAllocation("root.queueF"));
+ assertEquals(expectedResourceWithCustomType,
+ queueConf.getQueueMaxContainerAllocation("root.queueG"));
+ assertEquals(Resources.createResource(1024, 8),
+ queueConf.getQueueMaxContainerAllocation("root.queueG.queueH"));
+
assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root"));
assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java
new file mode 100644
index 0000000..2416d8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test Application master service using Fair scheduler.
+ */
+public class TestApplicationMasterServiceWithFS {
+
+ private static final Log LOG =
+ LogFactory.getLog(TestApplicationMasterServiceWithFS.class);
+
+ private static final int GB = 1024;
+ private static final int MEMORY_ALLOCATION = 3 * GB;
+ private static final String TEST_FOLDER = "test-queues";
+ private AllocateResponse allocateResponse;
+ private static YarnConfiguration configuration;
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ String allocFile =
+ GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath();
+
+ configuration = new YarnConfiguration();
+ configuration.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+ ResourceScheduler.class);
+ configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
+
+ PrintWriter out = new PrintWriter(new FileWriter(allocFile));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\">");
+ out.println(
+ " <maxContainerAllocation>2048 mb 1 vcores</maxContainerAllocation>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueB\">");
+ out.println(
+ " <maxContainerAllocation>3072 mb 1 vcores</maxContainerAllocation>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueC\">");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+ }
+
+ @AfterClass
+ public static void teardown(){
+ File allocFile = GenericTestUtils.getTestDir(TEST_FOLDER);
+ allocFile.delete();
+ }
+
+ @Test(timeout = 3000000)
+ public void testQueueLevelContainerAllocationFail() throws Exception {
+ MockRM rm = new MockRM(configuration);
+ rm.start();
+
+ // Register node1
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+
+ // Submit an application
+ RMApp app1 = rm.submitApp(2 * GB, "queueA");
+
+ // kick the scheduling
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ am1.registerAppAttempt();
+
+ am1.addRequests(new String[] { "127.0.0.1" }, MEMORY_ALLOCATION, 1, 1);
+ try {
+ allocateResponse = am1.schedule(); // send the request
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof InvalidResourceRequestException);
+ Assert.assertEquals(
+ InvalidResourceRequestException.InvalidResourceType.GREATER_THEN_MAX_ALLOCATION,
+ ((InvalidResourceRequestException) e).getInvalidResourceType());
+
+ } finally {
+ rm.stop();
+ }
+ }
+
+ @Test(timeout = 3000000)
+ public void testQueueLevelContainerAllocationSuccess() throws Exception {
+ testFairSchedulerContainerAllocationSuccess("queueB");
+ }
+
+ @Test(timeout = 3000000)
+ public void testSchedulerLevelContainerAllocationSuccess() throws Exception {
+ testFairSchedulerContainerAllocationSuccess("queueC");
+ }
+
+ private void testFairSchedulerContainerAllocationSuccess(String queueName)
+ throws Exception {
+ MockRM rm = new MockRM(configuration);
+ rm.start();
+
+ // Register node1
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+
+ // Submit an application
+ RMApp app1 = rm.submitApp(2 * GB, queueName);
+
+ // kick the scheduling
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ am1.registerAppAttempt();
+
+ am1.addRequests(new String[] { "127.0.0.1" }, MEMORY_ALLOCATION, 1, 1);
+
+ allocateResponse = am1.schedule(); // send the request
+ ((FairScheduler) rm.getResourceScheduler()).update();
+
+ // kick the scheduler
+ nm1.nodeHeartbeat(true);
+ GenericTestUtils.waitFor(() -> {
+ LOG.info("Waiting for containers to be created for app 1");
+ try {
+ allocateResponse = am1.schedule();
+ } catch (Exception e) {
+ Assert.fail("Allocation should be successful");
+ }
+ return allocateResponse.getAllocatedContainers().size() > 0;
+ }, 1000, 10000);
+
+ Container allocatedContainer =
+ allocateResponse.getAllocatedContainers().get(0);
+ Assert.assertEquals(MEMORY_ALLOCATION,
+ allocatedContainer.getResource().getMemorySize());
+ Assert.assertEquals(1, allocatedContainer.getResource().getVirtualCores());
+ rm.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 9120d3a..dbf4b2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -198,8 +199,6 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
- // TESTS
-
@SuppressWarnings("deprecation")
@Test(timeout=2000)
public void testLoadConfigurationOnInitialize() throws IOException {
@@ -338,6 +337,111 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ @Test
+ public void testQueueMaximumCapacityAllocations() throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ int tooHighQueueAllocation = RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE +1;
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\">");
+ out.println(
+ " <maxContainerAllocation>512 mb 1 vcores</maxContainerAllocation>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueB\">");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueC\">");
+ out.println(
+ " <maxContainerAllocation>2048 mb 3 vcores</maxContainerAllocation>");
+ out.println(" <queue name=\"queueD\">");
+ out.println(" </queue>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueE\">");
+ out.println(" <maxContainerAllocation>" + tooHighQueueAllocation
+ + " mb 1 vcores</maxContainerAllocation>");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+
+ Assert.assertEquals(1, scheduler.getMaximumResourceCapability("root.queueA")
+ .getVirtualCores());
+ Assert.assertEquals(512,
+ scheduler.getMaximumResourceCapability("root.queueA").getMemorySize());
+
+ Assert.assertEquals(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ scheduler.getMaximumResourceCapability("root.queueB")
+ .getVirtualCores());
+ Assert.assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE,
+ scheduler.getMaximumResourceCapability("root.queueB").getMemorySize());
+
+ Assert.assertEquals(3, scheduler.getMaximumResourceCapability("root.queueC")
+ .getVirtualCores());
+ Assert.assertEquals(2048,
+ scheduler.getMaximumResourceCapability("root.queueC").getMemorySize());
+
+ Assert.assertEquals(3, scheduler
+ .getMaximumResourceCapability("root.queueC.queueD").getVirtualCores());
+ Assert.assertEquals(2048, scheduler
+ .getMaximumResourceCapability("root.queueC.queueD").getMemorySize());
+
+ Assert.assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, scheduler
+ .getMaximumResourceCapability("root.queueE").getMemorySize());
+ }
+
+ @Test
+ public void testNormalizationUsingQueueMaximumAllocation()
+ throws IOException {
+
+ int queueMaxAllocation = 4096;
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\">");
+ out.println(" <maxContainerAllocation>" + queueMaxAllocation
+ + " mb 1 vcores" + "</maxContainerAllocation>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueB\">");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ allocateAppAttempt("root.queueA", 1, queueMaxAllocation + 1024);
+ allocateAppAttempt("root.queueB", 2,
+ RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE + 1024);
+
+ scheduler.update();
+ FSQueue queueToCheckA = scheduler.getQueueManager().getQueue("root.queueA");
+ FSQueue queueToCheckB = scheduler.getQueueManager().getQueue("root.queueB");
+
+ assertEquals(queueMaxAllocation, queueToCheckA.getDemand().getMemorySize());
+ assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE,
+ queueToCheckB.getDemand().getMemorySize());
+ }
+
+ private void allocateAppAttempt(String queueName, int id, int memorySize) {
+ ApplicationAttemptId id11 = createAppAttemptId(id, id);
+ createMockRMApp(id11);
+ scheduler.addApplication(id11.getApplicationId(), queueName, "user1",
+ false);
+ scheduler.addApplicationAttempt(id11, false, false);
+ List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
+ ResourceRequest request1 =
+ createResourceRequest(memorySize, ResourceRequest.ANY, 1, 1, true);
+ ask1.add(request1);
+ scheduler.allocate(id11, ask1, null, new ArrayList<ContainerId>(), null,
+ null, NULL_UPDATE_REQUESTS);
+ }
+
/**
* Test fair shares when max resources are set but are too high to impact
* the shares.
@@ -1316,8 +1420,9 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// New node satisfies resource request
scheduler.update();
scheduler.handle(new NodeUpdateSchedulerEvent(node4));
- assertEquals(10240, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE,
+ scheduler.getQueueManager().getQueue("queue1").getResourceUsage()
+ .getMemorySize());
scheduler.handle(new NodeUpdateSchedulerEvent(node1));
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -4099,12 +4204,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
- RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(10240, 10),
- 1, "127.0.0.1");
- RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(10240, 10),
- 2, "127.0.0.2");
+ RMNode node1 = MockNodes.newNodeInfo(1,
+ Resources.createResource(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 10),
+ 1, "127.0.0.1");
+ RMNode node2 = MockNodes.newNodeInfo(1,
+ Resources.createResource(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 10),
+ 2, "127.0.0.2");
RMNode node3 =
MockNodes.newNodeInfo(1, Resources.createResource(5120, 5),
3, "127.0.0.3");
@@ -4122,10 +4227,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
true);
Resource amResource1 = Resource.newInstance(1024, 1);
Resource amResource2 = Resource.newInstance(1024, 1);
- Resource amResource3 = Resource.newInstance(10240, 1);
+ Resource amResource3 =
+ Resource.newInstance(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1);
Resource amResource4 = Resource.newInstance(5120, 1);
Resource amResource5 = Resource.newInstance(1024, 1);
- Resource amResource6 = Resource.newInstance(10240, 1);
+ Resource amResource6 =
+ Resource.newInstance(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1);
Resource amResource7 = Resource.newInstance(1024, 1);
Resource amResource8 = Resource.newInstance(1024, 1);
int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();
@@ -4159,7 +4266,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ApplicationAttemptId attId3 = createAppAttemptId(3, 1);
createApplicationWithAMResource(attId3, "queue1", "user1", amResource3);
- createSchedulingRequestExistingApplication(10240, 1, amPriority, attId3);
+ createSchedulingRequestExistingApplication(
+ RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1, amPriority, attId3);
FSAppAttempt app3 = scheduler.getSchedulerApp(attId3);
scheduler.update();
// app3 reserves a container on node1 because node1's available resource
@@ -4233,7 +4341,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ApplicationAttemptId attId6 = createAppAttemptId(6, 1);
createApplicationWithAMResource(attId6, "queue1", "user1", amResource6);
- createSchedulingRequestExistingApplication(10240, 1, amPriority, attId6);
+ createSchedulingRequestExistingApplication(
+ RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1, amPriority, attId6);
FSAppAttempt app6 = scheduler.getSchedulerApp(attId6);
scheduler.update();
// app6 can't reserve a container on node1 because
@@ -4322,7 +4431,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// app6 turns the reservation into an allocation on node2.
scheduler.handle(updateE2);
assertEquals("Application6's AM requests 10240 MB memory",
- 10240, app6.getAMResource().getMemorySize());
+ RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE,
+ app6.getAMResource().getMemorySize());
assertEquals("Application6's AM should be running",
1, app6.getLiveContainers().size());
assertEquals("Queue1's AM resource usage should be 11264 MB memory",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
index f1afe69..db81613 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
@@ -60,9 +60,11 @@ class AllocationFileQueue {
() -> AllocationFileWriter
.createNumberSupplier(properties.getFairSharePreemptionTimeout()));
AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold",
+ () -> AllocationFileWriter.createNumberSupplier(
+ properties.getFairSharePreemptionThreshold()));
+ AllocationFileWriter.addIfPresent(pw, "maxContainerAllocation",
() -> AllocationFileWriter
- .createNumberSupplier(
- properties.getFairSharePreemptionThreshold()));
+ .createNumberSupplier(properties.getMaxContainerAllocation()));
printEndTag(pw);
pw.close();
return sw.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
index a2faf1d..176024e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
@@ -94,6 +94,12 @@ public abstract class AllocationFileQueueBuilder {
return this;
}
+ public AllocationFileQueueBuilder maxContainerAllocation(
+ String maxContainerAllocation) {
+ this.queuePropertiesBuilder.maxContainerAllocation(maxContainerAllocation);
+ return this;
+ }
+
public AllocationFileQueueBuilder subQueue(String queueName) {
if (this instanceof AllocationFileSimpleQueueBuilder) {
return new AllocationFileSubQueueBuilder(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
index 2c01144..0a0f330 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
@@ -33,6 +33,7 @@ public class AllocationFileQueueProperties {
private final String maxChildResources;
private final Integer fairSharePreemptionTimeout;
private final Double fairSharePreemptionThreshold;
+ private final String maxContainerAllocation;
AllocationFileQueueProperties(Builder builder) {
this.queueName = builder.queueName;
@@ -48,6 +49,7 @@ public class AllocationFileQueueProperties {
this.maxChildResources = builder.maxChildResources;
this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout;
this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold;
+ this.maxContainerAllocation = builder.maxContainerAllocation;
}
public String getQueueName() {
@@ -102,6 +104,10 @@ public class AllocationFileQueueProperties {
return fairSharePreemptionThreshold;
}
+ public String getMaxContainerAllocation() {
+ return maxContainerAllocation;
+ }
+
/**
* Builder class for {@link AllocationFileQueueProperties}.
*/
@@ -119,6 +125,7 @@ public class AllocationFileQueueProperties {
private String maxChildResources;
private Integer fairSharePreemptionTimeout;
private Double fairSharePreemptionThreshold;
+ private String maxContainerAllocation;
Builder() {
}
@@ -167,6 +174,11 @@ public class AllocationFileQueueProperties {
return this;
}
+ public Builder maxContainerAllocation(String maxContainerAllocation) {
+ this.maxContainerAllocation = maxContainerAllocation;
+ return this;
+ }
+
public Builder minSharePreemptionTimeout(
Integer minSharePreemptionTimeout) {
this.minSharePreemptionTimeout = minSharePreemptionTimeout;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index b5bcbf5..5f9e779 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -90,6 +90,8 @@ The allocation file must be in XML format. The format contains five types of ele
* **maxResources**: maximum resources a queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. A queue will not be assigned a container that would put its aggregate usage over this limit.
+ * **maxContainerAllocation**: maximum resources a queue can allocate for a single container, expressed in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. If the property is not set it's value is inherited from a parent queue. It's default value is **yarn.scheduler.maximum-allocation-mb**. Cannot be higher than **maxResources**. This property is invalid for root queue.
+
* **maxChildResources**: maximum resources an ad hoc child queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxRunningApps**: limit the number of apps from the queue to run at once
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-13936. Multipart upload to
HDFS to support 0 byte upload. Contributed by Ewan Higgs.
Posted by in...@apache.org.
HDFS-13936. Multipart upload to HDFS to support 0 byte upload. Contributed by Ewan Higgs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fab6886
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fab6886
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fab6886
Branch: refs/heads/HDFS-13532
Commit: 6fab6886f652492573734b832ca0375459a82775
Parents: a383ac4
Author: Ewan Higgs <eh...@apache.org>
Authored: Tue Oct 2 14:03:28 2018 +0200
Committer: Ewan Higgs <eh...@apache.org>
Committed: Tue Oct 2 14:05:35 2018 +0200
----------------------------------------------------------------------
.../hadoop/fs/FileSystemMultipartUploader.java | 25 +++++++++++++++-----
.../AbstractContractMultipartUploaderTest.java | 22 +++++++++++++++++
2 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
index 690194d..94c7861 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
@@ -100,6 +100,14 @@ public class FileSystemMultipartUploader extends MultipartUploader {
return fs.getPathHandle(status);
}
+ private long totalPartsLen(List<Path> partHandles) throws IOException {
+ long totalLen = 0;
+ for (Path p: partHandles) {
+ totalLen += fs.getFileStatus(p).getLen();
+ }
+ return totalLen;
+ }
+
@Override
@SuppressWarnings("deprecation") // rename w/ OVERWRITE
public PathHandle complete(Path filePath,
@@ -127,12 +135,17 @@ public class FileSystemMultipartUploader extends MultipartUploader {
.collect(Collectors.toList());
Path collectorPath = createCollectorPath(filePath);
- Path filePathInsideCollector = mergePaths(collectorPath,
- new Path(Path.SEPARATOR + filePath.getName()));
- fs.create(filePathInsideCollector).close();
- fs.concat(filePathInsideCollector,
- partHandles.toArray(new Path[handles.size()]));
- fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE);
+ boolean emptyFile = totalPartsLen(partHandles) == 0;
+ if (emptyFile) {
+ fs.create(filePath).close();
+ } else {
+ Path filePathInsideCollector = mergePaths(collectorPath,
+ new Path(Path.SEPARATOR + filePath.getName()));
+ fs.create(filePathInsideCollector).close();
+ fs.concat(filePathInsideCollector,
+ partHandles.toArray(new Path[handles.size()]));
+ fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE);
+ }
fs.delete(collectorPath, true);
return getPathHandle(filePath);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
index 85a6861..7cee5a6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
@@ -165,6 +165,28 @@ public abstract class AbstractContractMultipartUploaderTest extends
}
/**
+ * Assert that a multipart upload is successful when a single empty part is
+ * uploaded.
+ * @throws Exception failure
+ */
+ @Test
+ public void testMultipartUploadEmptyPart() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path file = path("testMultipartUpload");
+ MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+ UploadHandle uploadHandle = mpu.initialize(file);
+ List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+ MessageDigest origDigest = DigestUtils.getMd5Digest();
+ byte[] payload = new byte[0];
+ origDigest.update(payload);
+ InputStream is = new ByteArrayInputStream(payload);
+ PartHandle partHandle = mpu.putPart(file, is, 0, uploadHandle,
+ payload.length);
+ partHandles.add(Pair.of(0, partHandle));
+ completeUpload(file, mpu, uploadHandle, partHandles, origDigest, 0);
+ }
+
+ /**
* Assert that a multipart upload is successful even when the parts are
* given in the reverse order.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-325. Add event watcher for
delete blocks command. Contributed by Lokesh Jain.
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index a129ed0..5b9bcbd 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -46,13 +47,14 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.ozShell.TestOzoneShell;
+import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.utils.MetadataStore;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
+import org.slf4j.event.Level;
import java.io.File;
import java.io.IOException;
@@ -64,8 +66,13 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import static java.lang.Math.max;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static org.apache.hadoop.hdds
.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
@@ -76,14 +83,16 @@ public class TestBlockDeletion {
private static OzoneConfiguration conf = null;
private static ObjectStore store;
private static MiniOzoneCluster cluster = null;
- private static ContainerSet dnContainerSet = null;
private static StorageContainerManager scm = null;
private static OzoneManager om = null;
private static Set<Long> containerIdsWithDeletedBlocks;
+ private static long maxTransactionId = 0;
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
+ GenericTestUtils.setLogLevel(DeletedBlockLogImpl.LOG, Level.DEBUG);
+ GenericTestUtils.setLogLevel(SCMBlockDeletingService.LOG, Level.DEBUG);
String path =
GenericTestUtils.getTempPath(TestOzoneShell.class.getSimpleName());
@@ -94,6 +103,11 @@ public class TestBlockDeletion {
TimeUnit.MILLISECONDS);
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 200,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL,
+ 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1)
@@ -101,15 +115,12 @@ public class TestBlockDeletion {
.build();
cluster.waitForClusterToBeReady();
store = OzoneClientFactory.getRpcClient(conf).getObjectStore();
- dnContainerSet = cluster.getHddsDatanodes().get(0)
- .getDatanodeStateMachine().getContainer().getContainerSet();
om = cluster.getOzoneManager();
scm = cluster.getStorageContainerManager();
containerIdsWithDeletedBlocks = new HashSet<>();
}
- @Test(timeout = 60000)
- @Ignore("Until delete background service is fixed.")
+ @Test
public void testBlockDeletion()
throws IOException, InterruptedException, TimeoutException {
String volumeName = UUID.randomUUID().toString();
@@ -154,7 +165,9 @@ public class TestBlockDeletion {
Assert
.assertTrue(
OzoneTestUtils.closeContainers(omKeyLocationInfoGroupList, scm));
- Thread.sleep(5000);
+
+ waitForDatanodeCommandRetry();
+ waitForDatanodeBlockDeletionStart();
// The blocks should be deleted in the DN.
Assert.assertTrue(verifyBlocksDeleted(omKeyLocationInfoGroupList));
@@ -169,11 +182,52 @@ public class TestBlockDeletion {
matchContainerTransactionIds();
// verify PENDING_DELETE_STATUS event is fired
- verifyBlockDeletionEvent();
+ verifyPendingDeleteEvent();
+
+ // Verify transactions committed
+ verifyTransactionsCommitted();
+ }
+
+ private void waitForDatanodeBlockDeletionStart()
+ throws TimeoutException, InterruptedException {
+ LogCapturer logCapturer =
+ LogCapturer.captureLogs(DeleteBlocksCommandHandler.LOG);
+ logCapturer.clearOutput();
+ GenericTestUtils.waitFor(() -> logCapturer.getOutput()
+ .contains("Start to delete container block"),
+ 500, 10000);
+ Thread.sleep(1000);
+ }
+
+ /**
+ * Waits for datanode command to be retried when datanode is dead.
+ */
+ private void waitForDatanodeCommandRetry()
+ throws TimeoutException, InterruptedException {
+ cluster.shutdownHddsDatanode(0);
+ LogCapturer logCapturer =
+ LogCapturer.captureLogs(RetriableDatanodeEventWatcher.LOG);
+ logCapturer.clearOutput();
+ GenericTestUtils.waitFor(() -> logCapturer.getOutput()
+ .contains("RetriableDatanodeCommand type=deleteBlocksCommand"),
+ 500, 5000);
+ cluster.restartHddsDatanode(0);
+ }
+
+ private void verifyTransactionsCommitted() throws IOException {
+ DeletedBlockLogImpl deletedBlockLog =
+ (DeletedBlockLogImpl) scm.getScmBlockManager().getDeletedBlockLog();
+ for (int txnID = 1; txnID <= maxTransactionId; txnID++) {
+ Assert.assertNull(
+ deletedBlockLog.getDeletedStore().get(Longs.toByteArray(txnID)));
+ }
}
- private void verifyBlockDeletionEvent()
+ private void verifyPendingDeleteEvent()
throws IOException, InterruptedException {
+ ContainerSet dnContainerSet =
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer().getContainerSet();
LogCapturer logCapturer =
LogCapturer.captureLogs(SCMBlockDeletingService.LOG);
// Create dummy container reports with deleteTransactionId set as 0
@@ -209,6 +263,9 @@ public class TestBlockDeletion {
}
private void matchContainerTransactionIds() throws IOException {
+ ContainerSet dnContainerSet =
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer().getContainerSet();
List<ContainerData> containerDataList = new ArrayList<>();
dnContainerSet.listContainer(0, 10000, containerDataList);
for (ContainerData containerData : containerDataList) {
@@ -216,6 +273,8 @@ public class TestBlockDeletion {
if (containerIdsWithDeletedBlocks.contains(containerId)) {
Assert.assertTrue(
scm.getContainerInfo(containerId).getDeleteTransactionId() > 0);
+ maxTransactionId = max(maxTransactionId,
+ scm.getContainerInfo(containerId).getDeleteTransactionId());
} else {
Assert.assertEquals(
scm.getContainerInfo(containerId).getDeleteTransactionId(), 0);
@@ -230,6 +289,9 @@ public class TestBlockDeletion {
private boolean verifyBlocksCreated(
List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
throws IOException {
+ ContainerSet dnContainerSet =
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer().getContainerSet();
return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
try {
MetadataStore db = BlockUtils.getDB((KeyValueContainerData)
@@ -245,6 +307,9 @@ public class TestBlockDeletion {
private boolean verifyBlocksDeleted(
List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
throws IOException {
+ ContainerSet dnContainerSet =
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getContainer().getContainerSet();
return OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
try {
MetadataStore db = BlockUtils.getDB((KeyValueContainerData)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDFS-13938. Add a missing break in
BaseTestHttpFSWith. Contributed by Siyao Meng.
Posted by in...@apache.org.
HDFS-13938. Add a missing break in BaseTestHttpFSWith. Contributed by Siyao Meng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/284aca8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/284aca8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/284aca8b
Branch: refs/heads/HDFS-13532
Commit: 284aca8b63cf4f7b84d290721fcbff5f5db78983
Parents: 5c8d907
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Sep 28 15:44:46 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Sep 28 15:44:46 2018 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/284aca8b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index c7dc6f8..6680976 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -1175,6 +1175,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
break;
case DISALLOW_SNAPSHOT_EXCEPTION:
testDisallowSnapshotException();
+ break;
case FILE_STATUS_ATTR:
testFileStatusAttr();
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-6456. Added config to set
default container runtimes. Contributed by Craig Condit
Posted by in...@apache.org.
YARN-6456. Added config to set default container runtimes.
Contributed by Craig Condit
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b237a0dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b237a0dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b237a0dd
Branch: refs/heads/HDFS-13532
Commit: b237a0dd44ab285941983648d7ef26b99b30d624
Parents: 184544e
Author: Eric Yang <ey...@apache.org>
Authored: Thu Sep 27 15:28:38 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Sep 27 15:31:18 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 ++
.../src/main/resources/yarn-default.xml | 13 +++
.../container/ContainerImpl.java | 3 +
.../launcher/ContainerLaunch.java | 1 +
.../resources/gpu/GpuResourceHandlerImpl.java | 3 +
.../runtime/DefaultLinuxContainerRuntime.java | 6 +-
.../runtime/DockerLinuxContainerRuntime.java | 23 +++--
.../resources/gpu/TestGpuResourceHandler.java | 9 +-
.../runtime/TestDockerContainerRuntime.java | 91 +++++++++++++++++++-
.../src/site/markdown/DockerContainers.md | 17 ++++
10 files changed, 159 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d69ae57..95861d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1882,9 +1882,17 @@ public class YarnConfiguration extends Configuration {
public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES
= {"default"};
+ /** Default runtime to be used. */
+ public static final String LINUX_CONTAINER_RUNTIME_TYPE =
+ LINUX_CONTAINER_RUNTIME_PREFIX + "type";
+
public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
+ /** Default docker image to be used. */
+ public static final String NM_DOCKER_IMAGE_NAME =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name";
+
/** Capabilities allowed (and added by default) for docker containers. **/
public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0700902..e6f7b37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1726,6 +1726,12 @@
</property>
<property>
+ <description>Default container runtime to use.</description>
+ <name>yarn.nodemanager.runtime.linux.type</name>
+ <value></value>
+ </property>
+
+ <property>
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
@@ -1736,6 +1742,13 @@
</property>
<property>
+ <description>Default docker image to be used when the docker runtime is
+ selected.</description>
+ <name>yarn.nodemanager.runtime.linux.docker.image-name</name>
+ <value></value>
+ </property>
+
+ <property>
<description>This configuration setting determines if privileged docker
containers are allowed on this cluster. Privileged containers are granted
the complete set of capabilities and are not subject to the limitations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index e4cbfdc..f88dfbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1544,6 +1544,7 @@ public class ContainerImpl implements Container {
// TODO: Add containerWorkDir to the deletion service.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
@@ -1584,6 +1585,7 @@ public class ContainerImpl implements Container {
// TODO: Add containerOuputDir to the deletion service.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
@@ -1858,6 +1860,7 @@ public class ContainerImpl implements Container {
}
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6347d4e..3fa73ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -799,6 +799,7 @@ public class ContainerLaunch implements Callable<Integer> {
}
// The Docker container may not have fully started, reap the container.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ conf,
container.getLaunchContext().getEnvironment())) {
reapDockerContainerNoPid(user);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 1184382..e25a6b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -48,6 +48,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
+ private Context nmContext;
private GpuResourceAllocator gpuAllocator;
private CGroupsHandler cGroupsHandler;
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -55,6 +56,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public GpuResourceHandlerImpl(Context nmContext,
CGroupsHandler cGroupsHandler,
PrivilegedOperationExecutor privilegedOperationExecutor) {
+ this.nmContext = nmContext;
this.cGroupsHandler = cGroupsHandler;
this.privilegedOperationExecutor = privilegedOperationExecutor;
gpuAllocator = new GpuResourceAllocator(nmContext);
@@ -102,6 +104,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
containerIdStr);
if (!DockerLinuxContainerRuntime.isDockerContainerRequested(
+ nmContext.getConf(),
container.getLaunchContext().getEnvironment())) {
// Write to devices cgroup only for non-docker container. The reason is
// docker engine runtime runc do the devices cgroups initialize in the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 2b8e8ad..82ca6d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -71,7 +72,10 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public boolean isRuntimeRequested(Map<String, String> env) {
String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
- return type == null || type.equals("default");
+ if (type == null) {
+ type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
+ }
+ return type == null || type.isEmpty() || type.equals("default");
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 55dfdc3..c86ed53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -235,6 +235,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private Context nmContext;
private DockerClient dockerClient;
private PrivilegedOperationExecutor privilegedOperationExecutor;
+ private String defaultImageName;
private Set<String> allowedNetworks = new HashSet<>();
private String defaultNetwork;
private CGroupsHandler cGroupsHandler;
@@ -254,17 +255,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
* called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker},
* this method will return true. Otherwise it will return false.
*
+ * @param daemonConf the NodeManager daemon configuration
* @param env the environment variable settings for the operation
* @return whether a Docker container is requested
*/
- public static boolean isDockerContainerRequested(
+ public static boolean isDockerContainerRequested(Configuration daemonConf,
Map<String, String> env) {
- if (env == null) {
- return false;
+ String type = (env == null)
+ ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
+ if (type == null) {
+ type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
}
-
- String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
-
return type != null && type.equals("docker");
}
@@ -312,6 +313,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
defaultROMounts.clear();
defaultRWMounts.clear();
defaultTmpfsMounts.clear();
+ defaultImageName = conf.getTrimmed(
+ YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
allowedNetworks.addAll(Arrays.asList(
conf.getTrimmedStrings(
YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
@@ -325,8 +328,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
+ " is not in the set of allowed networks: " + allowedNetworks;
if (LOG.isWarnEnabled()) {
- LOG.warn(message + ". Please check "
- + "configuration");
+ LOG.warn(message + ". Please check configuration");
}
throw new ContainerExecutionException(message);
@@ -369,7 +371,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public boolean isRuntimeRequested(Map<String, String> env) {
- return isDockerContainerRequested(env);
+ return isDockerContainerRequested(conf, env);
}
private Set<String> getDockerCapabilitiesFromConf() throws
@@ -789,6 +791,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
boolean useEntryPoint = checkUseEntryPoint(environment);
+ if (imageName == null || imageName.isEmpty()) {
+ imageName = defaultImageName;
+ }
if(network == null || network.isEmpty()) {
network = defaultNetwork;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 9a8a4c9..301c3e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -80,8 +80,11 @@ public class TestGpuResourceHandler {
mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class);
mockNMStateStore = mock(NMStateStoreService.class);
+ Configuration conf = new Configuration();
+
Context nmctx = mock(Context.class);
when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore);
+ when(nmctx.getConf()).thenReturn(conf);
runningContainersMap = new ConcurrentHashMap<>();
when(nmctx.getContainers()).thenReturn(runningContainersMap);
@@ -347,15 +350,17 @@ public class TestGpuResourceHandler {
public void testAllocationStoredWithNULLStateStore() throws Exception {
NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class);
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
+
Context nmnctx = mock(Context.class);
when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore);
+ when(nmnctx.getConf()).thenReturn(conf);
GpuResourceHandlerImpl gpuNULLStateResourceHandler =
new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler,
mockPrivilegedExecutor);
- Configuration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
GpuDiscoverer.getInstance().initialize(conf);
gpuNULLStateResourceHandler.bootstrap(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 4464496..a880ba9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -309,11 +309,45 @@ public class TestDockerContainerRuntime {
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
Assert.assertEquals(false, DockerLinuxContainerRuntime
- .isDockerContainerRequested(null));
+ .isDockerContainerRequested(conf, null));
Assert.assertEquals(true, DockerLinuxContainerRuntime
- .isDockerContainerRequested(envDockerType));
+ .isDockerContainerRequested(conf, envDockerType));
Assert.assertEquals(false, DockerLinuxContainerRuntime
- .isDockerContainerRequested(envOtherType));
+ .isDockerContainerRequested(conf, envOtherType));
+ }
+
+ @Test
+ public void testSelectDockerContainerTypeWithDockerAsDefault() {
+ Map<String, String> envDockerType = new HashMap<>();
+ Map<String, String> envOtherType = new HashMap<>();
+
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker");
+ envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+ envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
+
+ Assert.assertEquals(true, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, null));
+ Assert.assertEquals(true, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envDockerType));
+ Assert.assertEquals(false, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envOtherType));
+ }
+
+ @Test
+ public void testSelectDockerContainerTypeWithDefaultSet() {
+ Map<String, String> envDockerType = new HashMap<>();
+ Map<String, String> envOtherType = new HashMap<>();
+
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default");
+ envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+ envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
+
+ Assert.assertEquals(false, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, null));
+ Assert.assertEquals(true, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envDockerType));
+ Assert.assertEquals(false, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envOtherType));
}
private PrivilegedOperation capturePrivilegedOperation()
@@ -422,6 +456,57 @@ public class TestDockerContainerRuntime {
}
@Test
+ public void testDockerContainerLaunchWithDefaultImage()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3");
+ env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE);
+
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf, nmContext);
+ runtime.launchContainer(builder.build());
+
+ PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+ List<String> args = op.getArguments();
+ String dockerCommandFile = args.get(11);
+
+ List<String> dockerCommands = Files.readAllLines(Paths.get(
+ dockerCommandFile), Charset.forName("UTF-8"));
+
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:1.2.3", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " name=container_e11_1518975676334_14532816_01_000001",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter));
+ }
+
+ @Test
public void testContainerLaunchWithUserRemapping()
throws ContainerExecutionException, PrivilegedOperationException,
IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 15105f3..17a335e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -115,6 +115,23 @@ The following properties should be set in yarn-site.xml:
</property>
<property>
+ <name>yarn.nodemanager.runtime.linux.type</name>
+ <value></value>
+ <description>
+ Optional. Sets the default container runtime to use.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.image-name</name>
+ <value></value>
+ <description>
+ Optional. Default docker image to be used when the docker runtime is
+ selected.
+ </description>
+ </property>
+
+ <property>
<name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
<value>host,none,bridge</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDFS-8196. Post enabled Erasure
Coding Policies on NameNode UI. Contributed by Kitti Nanasi and Kai Sasaki.
Posted by in...@apache.org.
HDFS-8196. Post enabled Erasure Coding Policies on NameNode UI. Contributed by Kitti Nanasi and Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2113500
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2113500
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2113500
Branch: refs/heads/HDFS-13532
Commit: e2113500df2479dd2d8f1d8a37fdb2bb16463181
Parents: 8598b49
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Sat Sep 29 00:53:35 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Sat Sep 29 00:53:35 2018 +0900
----------------------------------------------------------------------
.../namenode/ErasureCodingPolicyManager.java | 6 +++
.../hdfs/server/namenode/FSNamesystem.java | 9 ++++
.../namenode/metrics/ECBlockGroupsMBean.java | 5 ++
.../src/main/webapps/hdfs/dfshealth.html | 4 +-
.../server/namenode/TestNameNodeMXBean.java | 49 ++++++++++++++++++++
5 files changed, 72 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index e7de05b..d2bf3af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
+import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -430,4 +431,9 @@ public final class ErasureCodingPolicyManager {
allPolicies =
policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
}
+
+ public String getEnabledPoliciesMetric() {
+ return StringUtils.join(", ",
+ enabledPoliciesByName.keySet());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1caf7c2..137dfb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4973,6 +4973,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return blockManager.getTotalECBlockGroups();
}
+ /**
+ * Get the enabled erasure coding policies separated with comma.
+ */
+ @Override // ECBlockGroupsMBean
+ @Metric({"EnabledEcPolicies", "Enabled erasure coding policies"})
+ public String getEnabledEcPolicies() {
+ return getErasureCodingPolicyManager().getEnabledPoliciesMetric();
+ }
+
@Override
public long getBlockDeletionStartTime() {
return startTime + blockManager.getStartupDelayBlockDeletionInMs();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
index 808fc62..e5d2a09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
@@ -61,4 +61,9 @@ public interface ECBlockGroupsMBean {
* Return total number of erasure coded block groups.
*/
long getTotalECBlockGroups();
+
+ /**
+ * @return the enabled erasure coding policies separated with comma.
+ */
+ String getEnabledEcPolicies();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 4495b99..f993ae7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -186,10 +186,12 @@
{/eq}
<tr><th>Block Deletion Start Time</th><td>{BlockDeletionStartTime|date_tostring}</td></tr>
{/fs}
-
{#fsn}
<tr><th>Last Checkpoint Time</th><td>{@if cond="{LastCheckpointTime} === 0"}Never{:else}{LastCheckpointTime|date_tostring}{/if}</td></tr>
{/fsn}
+{#ecstat}
+ <tr><th>Enabled Erasure Coding Policies</th><td>{EnabledEcPolicies}</td></tr>
+{/ecstat}
</table>
<div class="page-header"><h1>NameNode Journal Status</h1></div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 3728420..07a4945 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
@@ -729,6 +730,46 @@ public class TestNameNodeMXBean {
}
@Test
+ public void testEnabledEcPoliciesMetric() throws Exception {
+ MiniDFSCluster cluster = null;
+ DistributedFileSystem fs = null;
+ try {
+ Configuration conf = new HdfsConfiguration();
+
+ ErasureCodingPolicy defaultPolicy =
+ StripedFileTestUtil.getDefaultECPolicy();
+ int dataBlocks = defaultPolicy.getNumDataUnits();
+ int parityBlocks = defaultPolicy.getNumParityUnits();
+ int totalSize = dataBlocks + parityBlocks;
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(totalSize).build();
+ fs = cluster.getFileSystem();
+
+ final String defaultPolicyName = defaultPolicy.getName();
+ final String rs104PolicyName = "RS-10-4-1024k";
+
+ assertEquals("Enabled EC policies metric should return with " +
+ "the default EC policy", defaultPolicyName,
+ getEnabledEcPoliciesMetric());
+
+ fs.enableErasureCodingPolicy(rs104PolicyName);
+ assertEquals("Enabled EC policies metric should return with " +
+ "both enabled policies separated by a comma",
+ rs104PolicyName + ", " + defaultPolicyName,
+ getEnabledEcPoliciesMetric());
+
+ fs.disableErasureCodingPolicy(defaultPolicyName);
+ fs.disableErasureCodingPolicy(rs104PolicyName);
+ assertEquals("Enabled EC policies metric should return with " +
+ "an empty string if there is no enabled policy",
+ "", getEnabledEcPoliciesMetric());
+ } finally {
+ fs.close();
+ cluster.shutdown();
+ }
+ }
+
+ @Test
public void testVerifyMissingBlockGroupsMetrics() throws Exception {
MiniDFSCluster cluster = null;
DistributedFileSystem fs = null;
@@ -968,4 +1009,12 @@ public class TestNameNodeMXBean {
assertEquals("Unexpected total ec block groups!",
expectedTotalECBlockGroups, totalECBlockGroups.longValue());
}
+
+ private String getEnabledEcPoliciesMetric() throws Exception {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=ECBlockGroupsState");
+ return (String) (mbs.getAttribute(mxbeanName,
+ "EnabledEcPolicies"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDFS-13927. Addendum: Improve
TestDataNodeMultipleRegistrations#testDNWithInvalidStorageWithHA wait.
Contributed by Ayush Saxena.
Posted by in...@apache.org.
HDFS-13927. Addendum: Improve TestDataNodeMultipleRegistrations#testDNWithInvalidStorageWithHA wait. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6275b428
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6275b428
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6275b428
Branch: refs/heads/HDFS-13532
Commit: 6275b428702b26e8b93e0f94573ccbef9a3b1faa
Parents: e5287a4
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Sep 26 09:50:09 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Sep 26 09:50:09 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/TestDataNodeMultipleRegistrations.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6275b428/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
index b2e3142..3a71d6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
@@ -310,7 +310,7 @@ public class TestDataNodeMultipleRegistrations {
}
return true;
}
- }, 500, 10000);
+ }, 500, 20000);
} finally {
cluster.shutdown();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-8760. [AMRMProxy] Fix
concurrent re-register due to YarnRM failover in AMRMClientRelayer.
Contributed by Botong Huang.
Posted by in...@apache.org.
YARN-8760. [AMRMProxy] Fix concurrent re-register due to YarnRM failover in AMRMClientRelayer. Contributed by Botong Huang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59d5af21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59d5af21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59d5af21
Branch: refs/heads/HDFS-13532
Commit: 59d5af21b7a8f52e8c89cbc2d25fe3d449b2657a
Parents: cc80ac2
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Mon Oct 1 13:12:38 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Mon Oct 1 13:12:38 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/AMRMClientRelayer.java | 25 ++++++++++++++++++--
.../yarn/server/TestAMRMClientRelayer.java | 25 ++++++++++++++++++++
.../amrmproxy/FederationInterceptor.java | 12 ++++++++++
3 files changed, 60 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d5af21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
index 2621d3e..ca045d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
@@ -237,6 +237,27 @@ public class AMRMClientRelayer extends AbstractService
return this.rmClient.registerApplicationMaster(request);
}
+ /**
+ * After an RM failover, there might be more than one
+ * allocate/finishApplicationMaster call thread (due to RPC timeout and retry)
+ * doing the auto re-register concurrently. As a result, we need to swallow
+ * the already register exception thrown by the new RM.
+ */
+ private void reRegisterApplicationMaster(
+ RegisterApplicationMasterRequest request)
+ throws YarnException, IOException {
+ try {
+ registerApplicationMaster(request);
+ } catch (InvalidApplicationMasterRequestException e) {
+ if (e.getMessage()
+ .contains(AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE)) {
+ LOG.info("Concurrent thread successfully re-registered, moving on.");
+ } else {
+ throw e;
+ }
+ }
+ }
+
@Override
public FinishApplicationMasterResponse finishApplicationMaster(
FinishApplicationMasterRequest request)
@@ -247,7 +268,7 @@ public class AMRMClientRelayer extends AbstractService
LOG.warn("Out of sync with RM " + rmId
+ " for " + this.appId + ", hence resyncing.");
// re register with RM
- registerApplicationMaster(this.amRegistrationRequest);
+ reRegisterApplicationMaster(this.amRegistrationRequest);
return finishApplicationMaster(request);
}
}
@@ -381,7 +402,7 @@ public class AMRMClientRelayer extends AbstractService
}
// re-register with RM, then retry allocate recursively
- registerApplicationMaster(this.amRegistrationRequest);
+ reRegisterApplicationMaster(this.amRegistrationRequest);
// Reset responseId after re-register
allocateRequest.setResponseId(0);
allocateResponse = allocate(allocateRequest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d5af21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
index 2c016d7..fa46960 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
@@ -64,6 +64,11 @@ public class TestAMRMClientRelayer {
// Whether this mockRM will throw failover exception upon next heartbeat
// from AM
private boolean failover = false;
+
+ // Whether this mockRM will throw application already registered exception
+ // upon next registerApplicationMaster call
+ private boolean throwAlreadyRegister = false;
+
private int responseIdReset = -1;
private List<ResourceRequest> lastAsk;
private List<ContainerId> lastRelease;
@@ -74,6 +79,11 @@ public class TestAMRMClientRelayer {
public RegisterApplicationMasterResponse registerApplicationMaster(
RegisterApplicationMasterRequest request)
throws YarnException, IOException {
+ if (this.throwAlreadyRegister) {
+ this.throwAlreadyRegister = false;
+ throw new InvalidApplicationMasterRequestException(
+ AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE + "appId");
+ }
return null;
}
@@ -118,6 +128,10 @@ public class TestAMRMClientRelayer {
this.failover = true;
}
+ public void setThrowAlreadyRegister() {
+ this.throwAlreadyRegister = true;
+ }
+
public void setResponseIdReset(int expectedResponseId) {
this.responseIdReset = expectedResponseId;
}
@@ -315,4 +329,15 @@ public class TestAMRMClientRelayer {
response = this.relayer.allocate(getAllocateRequest());
Assert.assertEquals(this.responseId + 1, response.getResponseId());
}
+
+ @Test
+ public void testConcurrentReregister() throws YarnException, IOException {
+
+ // Set RM restart and failover flag
+ this.mockAMS.setFailoverFlag();
+
+ this.mockAMS.setThrowAlreadyRegister();
+
+ relayer.finishApplicationMaster(null);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d5af21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index c02296d..4267945 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -184,6 +184,9 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
*/
private volatile boolean justRecovered;
+ /** if true, allocate will be no-op, skipping actual processing. */
+ private volatile boolean finishAMCalled;
+
/**
* Used to keep track of the container Id and the sub cluster RM that created
* the container, so that we know which sub-cluster to forward later requests
@@ -230,6 +233,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
this.amRegistrationRequest = null;
this.amRegistrationResponse = null;
this.justRecovered = false;
+ this.finishAMCalled = false;
}
/**
@@ -576,6 +580,12 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
+ ". AM should re-register and full re-send pending requests.");
}
+ if (this.finishAMCalled) {
+ LOG.warn("FinishApplicationMaster already called by {}, skip heartbeat "
+ + "processing and return dummy response" + this.attemptId);
+ return RECORD_FACTORY.newRecordInstance(AllocateResponse.class);
+ }
+
// Check responseId and handle duplicate heartbeat exactly same as RM
synchronized (this.lastAllocateResponseLock) {
LOG.info("Heartbeat from " + this.attemptId + " with responseId "
@@ -664,6 +674,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
FinishApplicationMasterRequest request)
throws YarnException, IOException {
+ this.finishAMCalled = true;
+
// TODO: consider adding batchFinishApplicationMaster in UAMPoolManager
boolean failedToUnRegister = false;
ExecutorCompletionService<FinishApplicationMasterResponseInfo> compSvc =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDDS-554. In XceiverClientSpi,
implement sendCommand(..) using sendCommandAsync(..). Contributed by
Tsz Wo Nicholas Sze.
Posted by in...@apache.org.
HDDS-554. In XceiverClientSpi, implement sendCommand(..) using sendCommandAsync(..). Contributed by Tsz Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f619504
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f619504
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f619504
Branch: refs/heads/HDFS-13532
Commit: 3f6195045ee826815956e33122943280f60b38f5
Parents: c62b440
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 26 09:15:25 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 26 09:16:23 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 23 --------------
.../hadoop/hdds/scm/XceiverClientRatis.java | 32 ++------------------
.../hadoop/hdds/scm/XceiverClientSpi.java | 10 ++++--
3 files changed, 10 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f619504/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index 3cdbc7c..d353e7a 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -120,29 +120,6 @@ public class XceiverClientGrpc extends XceiverClientSpi {
return pipeline;
}
- @Override
- public ContainerCommandResponseProto sendCommand(
- ContainerCommandRequestProto request) throws IOException {
- try {
- return sendCommandAsync(request).get();
- } catch (ExecutionException | InterruptedException e) {
- /**
- * In case the grpc channel handler throws an exception,
- * the exception thrown will be wrapped within {@link ExecutionException}.
- * Unwarpping here so that original exception gets passed
- * to to the client.
- */
- if (e instanceof ExecutionException) {
- Throwable cause = e.getCause();
- if (cause instanceof IOException) {
- throw (IOException) cause;
- }
- }
- throw new IOException(
- "Unexpected exception during execution:" + e.getMessage());
- }
- }
-
/**
* Sends a given command to server gets a waitable future back.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f619504/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 4c4de7f..0d301d9 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdds.scm;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.ratis.retry.RetryPolicy;
@@ -52,7 +51,6 @@ import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -183,20 +181,8 @@ public final class XceiverClientRatis extends XceiverClientSpi {
return Objects.requireNonNull(client.get(), "client is null");
}
- private RaftClientReply sendRequest(ContainerCommandRequestProto request)
- throws IOException {
- boolean isReadOnlyRequest = HddsUtils.isReadOnly(request);
- ByteString byteString = request.toByteString();
- LOG.debug("sendCommand {} {}", isReadOnlyRequest, request);
- final RaftClientReply reply = isReadOnlyRequest ?
- getClient().sendReadOnly(() -> byteString) :
- getClient().send(() -> byteString);
- LOG.debug("reply {} {}", isReadOnlyRequest, reply);
- return reply;
- }
-
private CompletableFuture<RaftClientReply> sendRequestAsync(
- ContainerCommandRequestProto request) throws IOException {
+ ContainerCommandRequestProto request) {
boolean isReadOnlyRequest = HddsUtils.isReadOnly(request);
ByteString byteString = request.toByteString();
LOG.debug("sendCommandAsync {} {}", isReadOnlyRequest, request);
@@ -204,19 +190,6 @@ public final class XceiverClientRatis extends XceiverClientSpi {
getClient().sendAsync(() -> byteString);
}
- @Override
- public ContainerCommandResponseProto sendCommand(
- ContainerCommandRequestProto request) throws IOException {
- final RaftClientReply reply = sendRequest(request);
- if (reply == null) {
- throw new IOException(
- String.format("Could not execute the request %s", request));
- }
- Preconditions.checkState(reply.isSuccess());
- return ContainerCommandResponseProto.parseFrom(
- reply.getMessage().getContent());
- }
-
/**
* Sends a given command to server gets a waitable future back.
*
@@ -226,8 +199,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
*/
@Override
public CompletableFuture<ContainerCommandResponseProto> sendCommandAsync(
- ContainerCommandRequestProto request)
- throws IOException, ExecutionException, InterruptedException {
+ ContainerCommandRequestProto request) {
return sendRequestAsync(request).whenComplete((reply, e) ->
LOG.debug("received reply {} for request: {} exception: {}", request,
reply, e))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f619504/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
index e8ef5c5..571d148 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java
@@ -95,8 +95,14 @@ public abstract class XceiverClientSpi implements Closeable {
* @return Response to the command
* @throws IOException
*/
- public abstract ContainerCommandResponseProto sendCommand(
- ContainerCommandRequestProto request) throws IOException;
+ public ContainerCommandResponseProto sendCommand(
+ ContainerCommandRequestProto request) throws IOException {
+ try {
+ return sendCommandAsync(request).get();
+ } catch (ExecutionException | InterruptedException e) {
+ throw new IOException("Failed to command " + request, e);
+ }
+ }
/**
* Sends a given command to server gets a waitable future back.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-370. Add and implement
following functions in SCMClientProtocolServer. Contributed by Ajay Kumar.
Posted by in...@apache.org.
HDDS-370. Add and implement following functions in SCMClientProtocolServer. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3928af98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3928af98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3928af98
Branch: refs/heads/HDFS-13532
Commit: 3928af983eb94d033e762739d3691c1618ba4f77
Parents: a4abf02
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Sep 25 12:50:51 2018 -0700
Committer: Ajay Kumar <aj...@apache.com>
Committed: Tue Sep 25 13:08:44 2018 -0700
----------------------------------------------------------------------
.../StorageContainerLocationProtocol.java | 16 +
...rLocationProtocolClientSideTranslatorPB.java | 42 +++
...rLocationProtocolServerSideTranslatorPB.java | 32 ++
.../StorageContainerLocationProtocol.proto | 27 ++
.../hdds/scm/server/SCMChillModeManager.java | 8 +-
.../scm/server/SCMClientProtocolServer.java | 22 ++
.../scm/server/StorageContainerManager.java | 8 +
.../ozone/TestStorageContainerManager.java | 151 ---------
.../hadoop/ozone/om/TestScmChillMode.java | 330 +++++++++++++++----
9 files changed, 414 insertions(+), 222 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index c55062b..e38077f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -133,4 +133,20 @@ public interface StorageContainerLocationProtocol {
* @throws IOException
*/
ScmInfo getScmInfo() throws IOException;
+
+ /**
+ * Check if SCM is in chill mode.
+ *
+ * @return Returns true if SCM is in chill mode else returns false.
+ * @throws IOException
+ */
+ boolean inChillMode() throws IOException;
+
+ /**
+ * Force SCM out of Chill mode.
+ *
+ * @return returns true if operation is successful.
+ * @throws IOException
+ */
+ boolean forceExitChillMode() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index 0441469..16819e9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -20,8 +20,12 @@ import com.google.common.base.Preconditions;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitChillModeRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ForceExitChillModeResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InChillModeRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.InChillModeResponseProto;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
@@ -317,6 +321,44 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
}
+ /**
+ * Check if SCM is in chill mode.
+ *
+ * @return Returns true if SCM is in chill mode else returns false.
+ * @throws IOException
+ */
+ @Override
+ public boolean inChillMode() throws IOException {
+ InChillModeRequestProto request =
+ InChillModeRequestProto.getDefaultInstance();
+ try {
+ InChillModeResponseProto resp = rpcProxy.inChillMode(
+ NULL_RPC_CONTROLLER, request);
+ return resp.getInChillMode();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ /**
+ * Force SCM out of Chill mode.
+ *
+ * @return returns true if operation is successful.
+ * @throws IOException
+ */
+ @Override
+ public boolean forceExitChillMode() throws IOException {
+ ForceExitChillModeRequestProto request =
+ ForceExitChillModeRequestProto.getDefaultInstance();
+ try {
+ ForceExitChillModeResponseProto resp = rpcProxy
+ .forceExitChillMode(NULL_RPC_CONTROLLER, request);
+ return resp.getExitedChillMode();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
@Override
public Object getUnderlyingProxyObject() {
return rpcProxy;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 9175ebf..d2723f0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -21,6 +21,14 @@ package org.apache.hadoop.ozone.protocolPB;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.InChillModeRequestProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.InChillModeResponseProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.ForceExitChillModeRequestProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerLocationProtocolProtos.ForceExitChillModeResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerWithPipelineResponseProto;
import org.apache.hadoop.hdds.scm.ScmInfo;
@@ -218,4 +226,28 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
}
}
+
+ @Override
+ public InChillModeResponseProto inChillMode(
+ RpcController controller,
+ InChillModeRequestProto request) throws ServiceException {
+ try {
+ return InChillModeResponseProto.newBuilder()
+ .setInChillMode(impl.inChillMode()).build();
+ } catch (IOException ex) {
+ throw new ServiceException(ex);
+ }
+ }
+
+ @Override
+ public ForceExitChillModeResponseProto forceExitChillMode(
+ RpcController controller, ForceExitChillModeRequestProto request)
+ throws ServiceException {
+ try {
+ return ForceExitChillModeResponseProto.newBuilder()
+ .setExitedChillMode(impl.forceExitChillMode()).build();
+ } catch (IOException ex) {
+ throw new ServiceException(ex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
index 68cc35f..fb01d6a 100644
--- a/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/StorageContainerLocationProtocol.proto
@@ -150,6 +150,21 @@ message PipelineResponseProto {
optional string errorMessage = 3;
}
+
+message InChillModeRequestProto {
+}
+
+message InChillModeResponseProto {
+ required bool inChillMode = 1;
+}
+
+message ForceExitChillModeRequestProto {
+}
+
+message ForceExitChillModeResponseProto {
+ required bool exitedChillMode = 1;
+}
+
/**
* Protocol used from an HDFS node to StorageContainerManager. See the request
* and response messages for details of the RPC calls.
@@ -209,4 +224,16 @@ service StorageContainerLocationProtocolService {
*/
rpc getScmInfo(GetScmInfoRequestProto)
returns (GetScmInfoRespsonseProto);
+
+ /**
+ * Checks if SCM is in ChillMode.
+ */
+ rpc inChillMode(InChillModeRequestProto)
+ returns (InChillModeResponseProto);
+
+ /**
+ * Returns information about SCM.
+ */
+ rpc forceExitChillMode(ForceExitChillModeRequestProto)
+ returns (ForceExitChillModeResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
index b35ac1b..3c1cc8f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
@@ -204,9 +204,11 @@ public class SCMChillModeManager implements
}
}
});
-
- LOG.info("SCM in chill mode. {} % containers have at least one reported "
- + "replica.", (containerWithMinReplicas.get() / maxContainer) * 100);
+ if(inChillMode.get()) {
+ LOG.info("SCM in chill mode. {} % containers have at least one"
+ + " reported replica.",
+ (containerWithMinReplicas.get() / maxContainer) * 100);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 3f1943c..66136f1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -302,6 +302,28 @@ public class SCMClientProtocolServer implements
}
/**
+ * Check if SCM is in chill mode.
+ *
+ * @return Returns true if SCM is in chill mode else returns false.
+ * @throws IOException
+ */
+ @Override
+ public boolean inChillMode() throws IOException {
+ return scm.isInChillMode();
+ }
+
+ /**
+ * Force SCM out of Chill mode.
+ *
+ * @return returns true if operation is successful.
+ * @throws IOException
+ */
+ @Override
+ public boolean forceExitChillMode() throws IOException {
+ return scm.exitChillMode();
+ }
+
+ /**
* Queries a list of Node that match a set of statuses.
*
* <p>For example, if the nodeStatuses is HEALTHY and RAFT_MEMBER, then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index b2cbc93..f491d30 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -887,6 +887,14 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return eventQueue;
}
+ /**
+ * Force SCM out of chill mode.
+ */
+ public boolean exitChillMode() {
+ scmChillModeManager.exitChillMode(eventQueue);
+ return true;
+ }
+
@VisibleForTesting
public double getCurrentContainerThreshold() {
return scmChillModeManager.getCurrentContainerThreshold();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 94ab6c8..fe53bcc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -18,13 +18,10 @@
package org.apache.hadoop.ozone;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.AtomicDouble;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -35,16 +32,11 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -52,13 +44,9 @@ import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLog;
import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
-import org.apache.hadoop.hdds.scm.container.ContainerMapping;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.server.SCMChillModeManager;
import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -69,7 +57,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -467,142 +454,4 @@ public class TestStorageContainerManager {
Assert.assertEquals(clusterId, scmInfo.getClusterId());
Assert.assertEquals(scmId, scmInfo.getScmId());
}
-
- @Test
- public void testSCMChillMode() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
- .setHbInterval(1000)
- .setNumDatanodes(3)
- .setStartDataNodes(false)
- .setHbProcessorInterval(500);
- MiniOzoneClusterImpl cluster = (MiniOzoneClusterImpl) builder.build();
- // Test1: Test chill mode when there are no containers in system.
- assertTrue(cluster.getStorageContainerManager().isInChillMode());
- cluster.startHddsDatanodes();
- cluster.waitForClusterToBeReady();
- assertFalse(cluster.getStorageContainerManager().isInChillMode());
-
- // Test2: Test chill mode when containers are there in system.
- // Create {numKeys} random names keys.
- TestStorageContainerManagerHelper helper =
- new TestStorageContainerManagerHelper(cluster, conf);
- Map<String, OmKeyInfo> keyLocations = helper.createKeys(100*2, 4096);
- final List<ContainerInfo> containers = cluster.getStorageContainerManager()
- .getScmContainerManager().getStateManager().getAllContainers();
- GenericTestUtils.waitFor(() -> {
- return containers.size() > 10;
- }, 100, 1000);
-
- // Removing some container to keep them open.
- containers.remove(0);
- containers.remove(1);
- containers.remove(2);
- containers.remove(3);
-
- // Close remaining containers
- ContainerMapping mapping = (ContainerMapping) cluster
- .getStorageContainerManager().getScmContainerManager();
- containers.forEach(c -> {
- try {
- mapping.updateContainerState(c.getContainerID(),
- HddsProtos.LifeCycleEvent.FINALIZE);
- mapping.updateContainerState(c.getContainerID(),
- LifeCycleEvent.CLOSE);
- } catch (IOException e) {
- LOG.info("Failed to change state of open containers.", e);
- }
- });
- cluster.stop();
-
- GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
- .captureLogs(SCMChillModeManager.getLogger());
- logCapturer.clearOutput();
- AtomicReference<MiniOzoneCluster> miniCluster = new AtomicReference<>();
- new Thread(() -> {
- try {
- miniCluster.set(builder.setStartDataNodes(false).build());
- } catch (IOException e) {
- fail("failed");
- }
- }).start();
-
- StorageContainerManager scm;
- GenericTestUtils.waitFor(() -> {
- return miniCluster.get() != null;
- }, 100, 1000 * 3);
-
- scm = miniCluster.get().getStorageContainerManager();
- assertTrue(scm.isInChillMode());
- assertFalse(logCapturer.getOutput().contains("SCM exiting chill mode."));
- assertTrue(scm.getCurrentContainerThreshold() == 0);
- AtomicDouble curThreshold = new AtomicDouble();
- AtomicDouble lastReportedThreshold = new AtomicDouble();
- for(HddsDatanodeService dn:miniCluster.get().getHddsDatanodes()){
- dn.start(null);
- GenericTestUtils.waitFor(() -> {
- curThreshold.set(scm.getCurrentContainerThreshold());
- return curThreshold.get() > lastReportedThreshold.get();
- }, 100, 1000 * 5);
- lastReportedThreshold.set(curThreshold.get());
- }
- double chillModeCutoff = conf
- .getDouble(HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT,
- HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT);
- assertTrue(scm.getCurrentContainerThreshold() >= chillModeCutoff);
- assertTrue(logCapturer.getOutput().contains("SCM exiting chill mode."));
- assertFalse(scm.isInChillMode());
- cluster.shutdown();
- }
-
- @Test
- public void testSCMChillModeRestrictedOp() throws Exception {
-
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
- OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB);
-
- MiniOzoneClusterImpl cluster = (MiniOzoneClusterImpl) MiniOzoneCluster
- .newBuilder(conf)
- .setHbInterval(1000)
- .setHbProcessorInterval(500)
- .setStartDataNodes(false)
- .build();
-
- StorageContainerManager scm = cluster.getStorageContainerManager();
- assertTrue(scm.isInChillMode());
-
- LambdaTestUtils.intercept(SCMException.class,
- "ChillModePrecheck failed for allocateContainer", () -> {
- scm.getClientProtocolServer()
- .allocateContainer(ReplicationType.STAND_ALONE,
- ReplicationFactor.ONE, "");
- });
-
- cluster.startHddsDatanodes();
- cluster.waitForClusterToBeReady();
- assertFalse(scm.isInChillMode());
-
- TestStorageContainerManagerHelper helper =
- new TestStorageContainerManagerHelper(cluster, conf);
- helper.createKeys(10, 4096);
- SCMClientProtocolServer clientProtocolServer = cluster
- .getStorageContainerManager().getClientProtocolServer();
- assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
- final List<ContainerInfo> containers = scm.getScmContainerManager()
- .getStateManager().getAllContainers();
- scm.getEventQueue().fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
- GenericTestUtils.waitFor(() -> {
- return clientProtocolServer.getChillModeStatus();
- }, 50, 1000 * 5);
- assertTrue(clientProtocolServer.getChillModeStatus());
-
- LambdaTestUtils.intercept(SCMException.class,
- "Open container " + containers.get(0).getContainerID() + " "
- + "doesn't have enough replicas to service this operation in Chill"
- + " mode.", () -> clientProtocolServer
- .getContainerWithPipeline(containers.get(0).getContainerID()));
- cluster.shutdown();
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3928af98/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
index 954fa0f..2a6921a 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
@@ -17,15 +17,30 @@
*/
package org.apache.hadoop.ozone.om;
+import com.google.common.util.concurrent.AtomicDouble;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.container.ContainerMapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdds.scm.server.SCMChillModeManager;
+import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneClusterImpl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.TestStorageContainerManagerHelper;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -41,7 +56,11 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
@@ -49,10 +68,14 @@ import static org.junit.Assert.fail;
*/
public class TestScmChillMode {
+ private final static Logger LOG = LoggerFactory
+ .getLogger(TestScmChillMode.class);
private static MiniOzoneCluster cluster = null;
private static MiniOzoneCluster.Builder builder = null;
private static OzoneConfiguration conf;
private static OzoneManager om;
+ private static StorageContainerLocationProtocolClientSideTranslatorPB
+ storageContainerLocationClient;
@Rule
@@ -77,6 +100,8 @@ public class TestScmChillMode {
cluster.startHddsDatanodes();
cluster.waitForClusterToBeReady();
om = cluster.getOzoneManager();
+ storageContainerLocationClient = cluster
+ .getStorageContainerLocationClient();
}
/**
@@ -85,7 +110,11 @@ public class TestScmChillMode {
@After
public void shutdown() {
if (cluster != null) {
- cluster.shutdown();
+ try {
+ cluster.shutdown();
+ } catch (Exception e) {
+ // do nothing.
+ }
}
}
@@ -93,79 +122,244 @@ public class TestScmChillMode {
public void testChillModeOperations() throws Exception {
final AtomicReference<MiniOzoneCluster> miniCluster =
new AtomicReference<>();
+ // Create {numKeys} random names keys.
+ TestStorageContainerManagerHelper helper =
+ new TestStorageContainerManagerHelper(cluster, conf);
+ Map<String, OmKeyInfo> keyLocations = helper.createKeys(100, 4096);
+ final List<ContainerInfo> containers = cluster
+ .getStorageContainerManager()
+ .getScmContainerManager().getStateManager().getAllContainers();
+ GenericTestUtils.waitFor(() -> {
+ return containers.size() > 10;
+ }, 100, 1000);
- try {
- // Create {numKeys} random names keys.
- TestStorageContainerManagerHelper helper =
- new TestStorageContainerManagerHelper(cluster, conf);
- Map<String, OmKeyInfo> keyLocations = helper.createKeys(100, 4096);
- final List<ContainerInfo> containers = cluster
- .getStorageContainerManager()
- .getScmContainerManager().getStateManager().getAllContainers();
- GenericTestUtils.waitFor(() -> {
- return containers.size() > 10;
- }, 100, 1000);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- String keyName = "key" + RandomStringUtils.randomNumeric(5);
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- OmKeyArgs keyArgs = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .setDataSize(1000)
- .build();
- OmVolumeArgs volArgs = new OmVolumeArgs.Builder()
- .setAdminName(adminName)
- .setCreationTime(Time.monotonicNow())
- .setQuotaInBytes(10000)
- .setVolume(volumeName)
- .setOwnerName(userName)
- .build();
- OmBucketInfo bucketInfo = new OmBucketInfo.Builder()
- .setBucketName(bucketName)
- .setIsVersionEnabled(false)
- .setVolumeName(volumeName)
- .build();
- om.createVolume(volArgs);
- om.createBucket(bucketInfo);
- om.openKey(keyArgs);
- //om.commitKey(keyArgs, 1);
-
- cluster.stop();
-
- new Thread(() -> {
- try {
- miniCluster.set(builder.build());
- } catch (IOException e) {
- fail("failed");
- }
- }).start();
-
- StorageContainerManager scm;
- GenericTestUtils.waitFor(() -> {
- return miniCluster.get() != null;
- }, 100, 1000 * 3);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ String keyName = "key" + RandomStringUtils.randomNumeric(5);
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(1000)
+ .build();
+ OmVolumeArgs volArgs = new OmVolumeArgs.Builder()
+ .setAdminName(adminName)
+ .setCreationTime(Time.monotonicNow())
+ .setQuotaInBytes(10000)
+ .setVolume(volumeName)
+ .setOwnerName(userName)
+ .build();
+ OmBucketInfo bucketInfo = new OmBucketInfo.Builder()
+ .setBucketName(bucketName)
+ .setIsVersionEnabled(false)
+ .setVolumeName(volumeName)
+ .build();
+ om.createVolume(volArgs);
+ om.createBucket(bucketInfo);
+ om.openKey(keyArgs);
+ //om.commitKey(keyArgs, 1);
+
+ cluster.stop();
+
+ new Thread(() -> {
+ try {
+ miniCluster.set(builder.build());
+ } catch (IOException e) {
+ fail("failed");
+ }
+ }).start();
- scm = miniCluster.get().getStorageContainerManager();
- Assert.assertTrue(scm.isInChillMode());
+ StorageContainerManager scm;
+ GenericTestUtils.waitFor(() -> {
+ return miniCluster.get() != null;
+ }, 100, 1000 * 3);
+ cluster = miniCluster.get();
- om = miniCluster.get().getOzoneManager();
+ scm = cluster.getStorageContainerManager();
+ Assert.assertTrue(scm.isInChillMode());
- LambdaTestUtils.intercept(OMException.class,
- "ChillModePrecheck failed for allocateBlock",
- () -> om.openKey(keyArgs));
+ om = miniCluster.get().getOzoneManager();
- } finally {
- if (miniCluster.get() != null) {
- try {
- miniCluster.get().shutdown();
- } catch (Exception e) {
- // do nothing.
- }
+ LambdaTestUtils.intercept(OMException.class,
+ "ChillModePrecheck failed for allocateBlock",
+ () -> om.openKey(keyArgs));
+ }
+
+ /**
+ * Tests inChillMode & forceExitChillMode api calls.
+ */
+ @Test
+ public void testIsScmInChillModeAndForceExit() throws Exception {
+ final AtomicReference<MiniOzoneCluster> miniCluster =
+ new AtomicReference<>();
+ // Test 1: SCM should be out of chill mode.
+ Assert.assertFalse(storageContainerLocationClient.inChillMode());
+ cluster.stop();
+ // Restart the cluster with same metadata dir.
+ new Thread(() -> {
+ try {
+ miniCluster.set(builder.build());
+ } catch (IOException e) {
+ Assert.fail("Cluster startup failed.");
}
+ }).start();
+ GenericTestUtils.waitFor(() -> {
+ return miniCluster.get() != null;
+ }, 10, 1000 * 3);
+ cluster = miniCluster.get();
+
+ // Test 2: Scm should be in chill mode as datanodes are not started yet.
+ storageContainerLocationClient = cluster
+ .getStorageContainerLocationClient();
+ Assert.assertTrue(storageContainerLocationClient.inChillMode());
+ // Force scm out of chill mode.
+ cluster.getStorageContainerManager().getClientProtocolServer()
+ .forceExitChillMode();
+ // Test 3: SCM should be out of chill mode.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return !cluster.getStorageContainerManager().getClientProtocolServer()
+ .inChillMode();
+ } catch (IOException e) {
+ Assert.fail("Cluster");
+ return false;
+ }
+ }, 10, 1000 * 5);
+
+ }
+
+ @Test
+ public void testSCMChillMode() throws Exception {
+ MiniOzoneCluster.Builder clusterBuilder = MiniOzoneCluster.newBuilder(conf)
+ .setHbInterval(1000)
+ .setNumDatanodes(3)
+ .setStartDataNodes(false)
+ .setHbProcessorInterval(500);
+ MiniOzoneClusterImpl miniCluster = (MiniOzoneClusterImpl) clusterBuilder
+ .build();
+ // Test1: Test chill mode when there are no containers in system.
+ assertTrue(miniCluster.getStorageContainerManager().isInChillMode());
+ miniCluster.startHddsDatanodes();
+ miniCluster.waitForClusterToBeReady();
+ assertFalse(miniCluster.getStorageContainerManager().isInChillMode());
+
+ // Test2: Test chill mode when containers are there in system.
+ // Create {numKeys} random names keys.
+ TestStorageContainerManagerHelper helper =
+ new TestStorageContainerManagerHelper(miniCluster, conf);
+ Map<String, OmKeyInfo> keyLocations = helper.createKeys(100 * 2, 4096);
+ final List<ContainerInfo> containers = miniCluster
+ .getStorageContainerManager().getScmContainerManager()
+ .getStateManager().getAllContainers();
+ GenericTestUtils.waitFor(() -> {
+ return containers.size() > 10;
+ }, 100, 1000 * 2);
+
+ // Removing some container to keep them open.
+ containers.remove(0);
+ containers.remove(1);
+ containers.remove(2);
+ containers.remove(3);
+
+ // Close remaining containers
+ ContainerMapping mapping = (ContainerMapping) miniCluster
+ .getStorageContainerManager().getScmContainerManager();
+ containers.forEach(c -> {
+ try {
+ mapping.updateContainerState(c.getContainerID(),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ mapping.updateContainerState(c.getContainerID(),
+ LifeCycleEvent.CLOSE);
+ } catch (IOException e) {
+ LOG.info("Failed to change state of open containers.", e);
+ }
+ });
+ miniCluster.stop();
+
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(SCMChillModeManager.getLogger());
+ logCapturer.clearOutput();
+ AtomicReference<MiniOzoneCluster> miniClusterOzone
+ = new AtomicReference<>();
+ new Thread(() -> {
+ try {
+ miniClusterOzone.set(clusterBuilder.setStartDataNodes(false).build());
+ } catch (IOException e) {
+ fail("failed");
+ }
+ }).start();
+
+ StorageContainerManager scm;
+ GenericTestUtils.waitFor(() -> {
+ return miniClusterOzone.get() != null;
+ }, 100, 1000 * 3);
+
+ miniCluster = (MiniOzoneClusterImpl) miniClusterOzone.get();
+
+ scm = miniCluster.getStorageContainerManager();
+ assertTrue(scm.isInChillMode());
+ assertFalse(logCapturer.getOutput().contains("SCM exiting chill mode."));
+ assertTrue(scm.getCurrentContainerThreshold() == 0);
+ AtomicDouble curThreshold = new AtomicDouble();
+ AtomicDouble lastReportedThreshold = new AtomicDouble();
+ for (HddsDatanodeService dn : miniCluster.getHddsDatanodes()) {
+ dn.start(null);
+ GenericTestUtils.waitFor(() -> {
+ curThreshold.set(scm.getCurrentContainerThreshold());
+ return curThreshold.get() > lastReportedThreshold.get();
+ }, 100, 1000 * 5);
+ lastReportedThreshold.set(curThreshold.get());
}
+ cluster = miniCluster;
+ double chillModeCutoff = conf
+ .getDouble(HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT,
+ HddsConfigKeys.HDDS_SCM_CHILLMODE_THRESHOLD_PCT_DEFAULT);
+ assertTrue(scm.getCurrentContainerThreshold() >= chillModeCutoff);
+ assertTrue(logCapturer.getOutput().contains("SCM exiting chill mode."));
+ assertFalse(scm.isInChillMode());
+ }
+
+ @Test
+ public void testSCMChillModeRestrictedOp() throws Exception {
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
+ OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB);
+ cluster.stop();
+ cluster = builder.build();
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ assertTrue(scm.isInChillMode());
+
+ LambdaTestUtils.intercept(SCMException.class,
+ "ChillModePrecheck failed for allocateContainer", () -> {
+ scm.getClientProtocolServer()
+ .allocateContainer(ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE, "");
+ });
+
+ cluster.startHddsDatanodes();
+ cluster.waitForClusterToBeReady();
+ assertFalse(scm.isInChillMode());
+
+ TestStorageContainerManagerHelper helper =
+ new TestStorageContainerManagerHelper(cluster, conf);
+ helper.createKeys(10, 4096);
+ SCMClientProtocolServer clientProtocolServer = cluster
+ .getStorageContainerManager().getClientProtocolServer();
+ assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
+ final List<ContainerInfo> containers = scm.getScmContainerManager()
+ .getStateManager().getAllContainers();
+ scm.getEventQueue().fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
+ GenericTestUtils.waitFor(() -> {
+ return clientProtocolServer.getChillModeStatus();
+ }, 50, 1000 * 5);
+ assertTrue(clientProtocolServer.getChillModeStatus());
+
+ LambdaTestUtils.intercept(SCMException.class,
+ "Open container " + containers.get(0).getContainerID() + " "
+ + "doesn't have enough replicas to service this operation in Chill"
+ + " mode.", () -> clientProtocolServer
+ .getContainerWithPipeline(containers.get(0).getContainerID()));
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDDS-325. Add event watcher for
delete blocks command. Contributed by Lokesh Jain.
Posted by in...@apache.org.
HDDS-325. Add event watcher for delete blocks command. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7ff8c05
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7ff8c05
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7ff8c05
Branch: refs/heads/HDFS-13532
Commit: f7ff8c051e4d4b5cd0eee1884e2a546de1f57793
Parents: fd6be58
Author: Nanda kumar <na...@apache.org>
Authored: Mon Oct 1 13:49:55 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Mon Oct 1 13:50:17 2018 +0530
----------------------------------------------------------------------
.../scm/container/common/helpers/Pipeline.java | 4 +
.../report/CommandStatusReportPublisher.java | 2 +-
.../common/statemachine/StateContext.java | 58 +++++++-----
.../CloseContainerCommandHandler.java | 3 +-
.../commandhandler/CommandHandler.java | 7 +-
.../DeleteBlocksCommandHandler.java | 43 +++++----
.../ReplicateContainerCommandHandler.java | 3 +-
.../states/endpoint/HeartbeatEndpointTask.java | 33 ++++++-
.../container/ozoneimpl/ContainerReader.java | 20 +++++
.../StorageContainerDatanodeProtocol.java | 13 ---
.../protocol/commands/CommandForDatanode.java | 8 +-
.../ozone/protocol/commands/CommandStatus.java | 41 +++++++--
.../commands/DeleteBlockCommandStatus.java | 92 ++++++++++++++++++++
...rDatanodeProtocolClientSideTranslatorPB.java | 17 ----
...rDatanodeProtocolServerSideTranslatorPB.java | 16 ----
.../StorageContainerDatanodeProtocol.proto | 11 +--
.../ozone/container/common/ScmTestMock.java | 24 ++---
.../common/report/TestReportPublisher.java | 4 +-
.../endpoint/TestHeartbeatEndpointTask.java | 12 +--
.../block/DatanodeDeletedBlockTransactions.java | 20 ++---
.../hdds/scm/block/DeletedBlockLogImpl.java | 24 ++++-
.../hdds/scm/block/SCMBlockDeletingService.java | 2 +-
.../scm/command/CommandStatusReportHandler.java | 10 ++-
.../hdds/scm/container/ContainerMapping.java | 19 ++--
.../hadoop/hdds/scm/events/SCMEvents.java | 19 ++--
.../server/SCMDatanodeHeartbeatDispatcher.java | 11 ++-
.../scm/server/SCMDatanodeProtocolServer.java | 22 -----
.../scm/server/StorageContainerManager.java | 12 +++
.../commands/RetriableDatanodeEventWatcher.java | 58 ++++++++++++
.../ozone/protocol/commands/package-info.java | 18 ++++
.../command/TestCommandStatusReportHandler.java | 12 +--
.../TestSCMDatanodeHeartbeatDispatcher.java | 2 +-
.../ozone/TestStorageContainerManager.java | 3 +
.../commandhandler/TestBlockDeletion.java | 83 ++++++++++++++++--
34 files changed, 503 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
index 777efa7..c36ca1f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
@@ -171,6 +171,10 @@ public class Pipeline {
public Map<String, DatanodeDetails> getDatanodes() {
return datanodes;
}
+
+ public boolean isEmpty() {
+ return datanodes.isEmpty();
+ }
/**
* Returns the leader host.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
index 4cf6321..4736857 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/CommandStatusReportPublisher.java
@@ -82,6 +82,6 @@ public class CommandStatusReportPublisher extends
map.remove(key);
}
});
- return builder.build();
+ return builder.getCmdStatusCount() > 0 ? builder.build() : null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 47c2492..12c196b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -22,6 +22,8 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineAction;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerAction;
@@ -35,8 +37,11 @@ import org.apache.hadoop.ozone.container.common.states.datanode
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus
.CommandStatusBuilder;
+import org.apache.hadoop.ozone.protocol.commands
+ .DeleteBlockCommandStatus.DeleteBlockCommandStatusBuilder;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import static java.lang.Math.min;
import static org.apache.hadoop.hdds.scm.HddsServerUtil.getScmHeartbeatInterval;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,6 +57,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
@@ -67,7 +73,7 @@ public class StateContext {
private final DatanodeStateMachine parent;
private final AtomicLong stateExecutionCount;
private final Configuration conf;
- private final Queue<GeneratedMessage> reports;
+ private final List<GeneratedMessage> reports;
private final Queue<ContainerAction> containerActions;
private final Queue<PipelineAction> pipelineActions;
private DatanodeStateMachine.DatanodeStates state;
@@ -174,19 +180,23 @@ public class StateContext {
* @param report report to be added
*/
public void addReport(GeneratedMessage report) {
- synchronized (reports) {
- reports.add(report);
+ if (report != null) {
+ synchronized (reports) {
+ reports.add(report);
+ }
}
}
/**
- * Returns the next report, or null if the report queue is empty.
+ * Adds the reports which could not be sent by heartbeat back to the
+ * reports list.
*
- * @return report
+ * @param reportsToPutBack list of reports which failed to be sent by
+ * heartbeat.
*/
- public GeneratedMessage getNextReport() {
+ public void putBackReports(List<GeneratedMessage> reportsToPutBack) {
synchronized (reports) {
- return reports.poll();
+ reports.addAll(0, reportsToPutBack);
}
}
@@ -207,19 +217,14 @@ public class StateContext {
* @return List<reports>
*/
public List<GeneratedMessage> getReports(int maxLimit) {
- List<GeneratedMessage> reportList = new ArrayList<>();
+ List<GeneratedMessage> reportsToReturn = new LinkedList<>();
synchronized (reports) {
- if (!reports.isEmpty()) {
- int size = reports.size();
- int limit = size > maxLimit ? maxLimit : size;
- for (int count = 0; count < limit; count++) {
- GeneratedMessage report = reports.poll();
- Preconditions.checkNotNull(report);
- reportList.add(report);
- }
- }
- return reportList;
+ List<GeneratedMessage> tempList = reports.subList(
+ 0, min(reports.size(), maxLimit));
+ reportsToReturn.addAll(tempList);
+ tempList.clear();
}
+ return reportsToReturn;
}
@@ -442,9 +447,14 @@ public class StateContext {
* @param cmd - {@link SCMCommand}.
*/
public void addCmdStatus(SCMCommand cmd) {
+ CommandStatusBuilder statusBuilder;
+ if (cmd.getType() == Type.deleteBlocksCommand) {
+ statusBuilder = new DeleteBlockCommandStatusBuilder();
+ } else {
+ statusBuilder = CommandStatusBuilder.newBuilder();
+ }
this.addCmdStatus(cmd.getId(),
- CommandStatusBuilder.newBuilder()
- .setCmdId(cmd.getId())
+ statusBuilder.setCmdId(cmd.getId())
.setStatus(Status.PENDING)
.setType(cmd.getType())
.build());
@@ -469,13 +479,13 @@ public class StateContext {
/**
* Updates status of a pending status command.
* @param cmdId command id
- * @param cmdExecuted SCMCommand
+ * @param cmdStatusUpdater Consumer to update command status.
* @return true if command status updated successfully else false.
*/
- public boolean updateCommandStatus(Long cmdId, boolean cmdExecuted) {
+ public boolean updateCommandStatus(Long cmdId,
+ Consumer<CommandStatus> cmdStatusUpdater) {
if(cmdStatusMap.containsKey(cmdId)) {
- cmdStatusMap.get(cmdId)
- .setStatus(cmdExecuted ? Status.EXECUTED : Status.FAILED);
+ cmdStatusUpdater.accept(cmdStatusMap.get(cmdId));
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index 0af591b..2c3db61 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -117,7 +117,8 @@ public class CloseContainerCommandHandler implements CommandHandler {
cmdExecuted = false;
}
} finally {
- updateCommandStatus(context, command, cmdExecuted, LOG);
+ updateCommandStatus(context, command,
+ (cmdStatus) -> cmdStatus.setStatus(cmdExecuted), LOG);
long endTime = Time.monotonicNow();
totalTime += endTime - startTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
index 71c25b5..1ea0ea8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
@@ -23,9 +23,12 @@ import org.apache.hadoop.ozone.container.common.statemachine
.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.slf4j.Logger;
+import java.util.function.Consumer;
+
/**
* Generic interface for handlers.
*/
@@ -63,8 +66,8 @@ public interface CommandHandler {
* Default implementation for updating command status.
*/
default void updateCommandStatus(StateContext context, SCMCommand command,
- boolean cmdExecuted, Logger log) {
- if (!context.updateCommandStatus(command.getId(), cmdExecuted)) {
+ Consumer<CommandStatus> cmdStatusUpdater, Logger log) {
+ if (!context.updateCommandStatus(command.getId(), cmdStatusUpdater)) {
log.debug("{} with Id:{} not found.", command.getType(),
command.getId());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index 430b0ef..aa63fb4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -39,11 +39,11 @@ import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
+import org.apache.hadoop.ozone.protocol.commands.DeleteBlockCommandStatus;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
@@ -54,6 +54,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
+import java.util.function.Consumer;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Result.CONTAINER_NOT_FOUND;
@@ -63,7 +64,7 @@ import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
*/
public class DeleteBlocksCommandHandler implements CommandHandler {
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(DeleteBlocksCommandHandler.class);
private final ContainerSet containerSet;
@@ -83,6 +84,7 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
StateContext context, SCMConnectionManager connectionManager) {
cmdExecuted = false;
long startTime = Time.monotonicNow();
+ ContainerBlocksDeletionACKProto blockDeletionACK = null;
try {
if (command.getType() != SCMCommandProto.Type.deleteBlocksCommand) {
LOG.warn("Skipping handling command, expected command "
@@ -144,31 +146,28 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
.setDnId(context.getParent().getDatanodeDetails()
.getUuid().toString());
});
- ContainerBlocksDeletionACKProto blockDeletionACK = resultBuilder.build();
+ blockDeletionACK = resultBuilder.build();
// Send ACK back to SCM as long as meta updated
// TODO Or we should wait until the blocks are actually deleted?
if (!containerBlocks.isEmpty()) {
- for (EndpointStateMachine endPoint : connectionManager.getValues()) {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sending following block deletion ACK to SCM");
- for (DeleteBlockTransactionResult result :
- blockDeletionACK.getResultsList()) {
- LOG.debug(result.getTxID() + " : " + result.getSuccess());
- }
- }
- endPoint.getEndPoint()
- .sendContainerBlocksDeletionACK(blockDeletionACK);
- } catch (IOException e) {
- LOG.error("Unable to send block deletion ACK to SCM {}",
- endPoint.getAddress().toString(), e);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending following block deletion ACK to SCM");
+ for (DeleteBlockTransactionResult result : blockDeletionACK
+ .getResultsList()) {
+ LOG.debug(result.getTxID() + " : " + result.getSuccess());
}
}
}
cmdExecuted = true;
} finally {
- updateCommandStatus(context, command, cmdExecuted, LOG);
+ final ContainerBlocksDeletionACKProto deleteAck =
+ blockDeletionACK;
+ Consumer<CommandStatus> statusUpdater = (cmdStatus) -> {
+ cmdStatus.setStatus(cmdExecuted);
+ ((DeleteBlockCommandStatus) cmdStatus).setBlocksDeletionAck(deleteAck);
+ };
+ updateCommandStatus(context, command, statusUpdater, LOG);
long endTime = Time.monotonicNow();
totalTime += endTime - startTime;
}
@@ -238,9 +237,9 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
}
}
- containerDB.put(DFSUtil.string2Bytes(
- OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()),
- Longs.toByteArray(delTX.getTxID()));
+ containerDB
+ .put(DFSUtil.string2Bytes(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX),
+ Longs.toByteArray(delTX.getTxID()));
containerData
.updateDeleteTransactionId(delTX.getTxID());
// update pending deletion blocks count in in-memory container status
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
index 09c379f..81d162d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -77,7 +77,8 @@ public class ReplicateContainerCommandHandler implements CommandHandler {
supervisor.addTask(replicationTask);
} finally {
- updateCommandStatus(context, command, true, LOG);
+ updateCommandStatus(context, command,
+ (cmdStatus) -> cmdStatus.setStatus(true), LOG);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 5769e6d..4fd72ec 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -54,6 +54,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.time.ZonedDateTime;
+import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -124,12 +125,12 @@ public class HeartbeatEndpointTask
@Override
public EndpointStateMachine.EndPointStates call() throws Exception {
rpcEndpoint.lock();
+ SCMHeartbeatRequestProto.Builder requestBuilder = null;
try {
Preconditions.checkState(this.datanodeDetailsProto != null);
- SCMHeartbeatRequestProto.Builder requestBuilder =
- SCMHeartbeatRequestProto.newBuilder()
- .setDatanodeDetails(datanodeDetailsProto);
+ requestBuilder = SCMHeartbeatRequestProto.newBuilder()
+ .setDatanodeDetails(datanodeDetailsProto);
addReports(requestBuilder);
addContainerActions(requestBuilder);
addPipelineActions(requestBuilder);
@@ -139,6 +140,8 @@ public class HeartbeatEndpointTask
rpcEndpoint.setLastSuccessfulHeartbeat(ZonedDateTime.now());
rpcEndpoint.zeroMissedCount();
} catch (IOException ex) {
+ // put back the reports which failed to be sent
+ putBackReports(requestBuilder);
rpcEndpoint.logIfNeeded(ex);
} finally {
rpcEndpoint.unlock();
@@ -146,6 +149,24 @@ public class HeartbeatEndpointTask
return rpcEndpoint.getState();
}
+ // TODO: Make it generic.
+ private void putBackReports(SCMHeartbeatRequestProto.Builder requestBuilder) {
+ List<GeneratedMessage> reports = new LinkedList<>();
+ if (requestBuilder.hasContainerReport()) {
+ reports.add(requestBuilder.getContainerReport());
+ }
+ if (requestBuilder.hasNodeReport()) {
+ reports.add(requestBuilder.getNodeReport());
+ }
+ if (requestBuilder.getCommandStatusReportsCount() != 0) {
+ for (GeneratedMessage msg : requestBuilder
+ .getCommandStatusReportsList()) {
+ reports.add(msg);
+ }
+ }
+ context.putBackReports(reports);
+ }
+
/**
* Adds all the available reports to heartbeat.
*
@@ -158,7 +179,11 @@ public class HeartbeatEndpointTask
SCMHeartbeatRequestProto.getDescriptor().getFields()) {
String heartbeatFieldName = descriptor.getMessageType().getFullName();
if (heartbeatFieldName.equals(reportName)) {
- requestBuilder.setField(descriptor, report);
+ if (descriptor.isRepeated()) {
+ requestBuilder.addRepeatedField(descriptor, report);
+ } else {
+ requestBuilder.setField(descriptor, report);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
index 7c986f0..c3a4126 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
@@ -19,10 +19,13 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -32,7 +35,10 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
+import org.apache.hadoop.utils.MetadataKeyFilters;
+import org.apache.hadoop.utils.MetadataStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -171,6 +177,20 @@ public class ContainerReader implements Runnable {
KeyValueContainerUtil.parseKVContainerData(kvContainerData, config);
KeyValueContainer kvContainer = new KeyValueContainer(
kvContainerData, config);
+ MetadataStore containerDB = BlockUtils.getDB(kvContainerData, config);
+ MetadataKeyFilters.KeyPrefixFilter filter =
+ new MetadataKeyFilters.KeyPrefixFilter()
+ .addFilter(OzoneConsts.DELETING_KEY_PREFIX);
+ int numPendingDeletionBlocks =
+ containerDB.getSequentialRangeKVs(null, Integer.MAX_VALUE, filter)
+ .size();
+ kvContainerData.incrPendingDeletionBlocks(numPendingDeletionBlocks);
+ byte[] delTxnId = containerDB.get(
+ DFSUtil.string2Bytes(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX));
+ if (delTxnId != null) {
+ kvContainerData
+ .updateDeleteTransactionId(Longs.fromByteArray(delTxnId));
+ }
containerSet.addContainer(kvContainer);
} else {
throw new StorageContainerException("Container File is corrupted. " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index 9296524..e3b5370 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -23,11 +23,6 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos
- .ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
@@ -77,12 +72,4 @@ public interface StorageContainerDatanodeProtocol {
ContainerReportsProto containerReportsRequestProto,
PipelineReportsProto pipelineReports) throws IOException;
- /**
- * Used by datanode to send block deletion ACK to SCM.
- * @param request block deletion transactions.
- * @return block deletion transaction response.
- * @throws IOException
- */
- ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- ContainerBlocksDeletionACKProto request) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
index 0c4964a..69337fb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.ozone.protocol.commands;
import java.util.UUID;
import com.google.protobuf.GeneratedMessage;
+import org.apache.hadoop.hdds.server.events.IdentifiableEventPayload;
/**
* Command for the datanode with the destination address.
*/
-public class CommandForDatanode<T extends GeneratedMessage> {
+public class CommandForDatanode<T extends GeneratedMessage> implements
+ IdentifiableEventPayload {
private final UUID datanodeId;
@@ -42,4 +44,8 @@ public class CommandForDatanode<T extends GeneratedMessage> {
public SCMCommand<T> getCommand() {
return command;
}
+
+ public long getId() {
+ return command.getId();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandStatus.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandStatus.java
index 32cf7c2..4b3ce84 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandStatus.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandStatus.java
@@ -35,6 +35,13 @@ public class CommandStatus {
private Status status;
private String msg;
+ CommandStatus(Type type, Long cmdId, Status status, String msg) {
+ this.type = type;
+ this.cmdId = cmdId;
+ this.status = status;
+ this.msg = msg;
+ }
+
public Type getType() {
return type;
}
@@ -60,6 +67,10 @@ public class CommandStatus {
this.status = status;
}
+ public void setStatus(boolean cmdExecuted) {
+ setStatus(cmdExecuted ? Status.EXECUTED : Status.FAILED);
+ }
+
/**
* Returns a CommandStatus from the protocol buffers.
*
@@ -72,7 +83,8 @@ public class CommandStatus {
.setCmdId(cmdStatusProto.getCmdId())
.setStatus(cmdStatusProto.getStatus())
.setType(cmdStatusProto.getType())
- .setMsg(cmdStatusProto.getMsg()).build();
+ .setMsg(cmdStatusProto.getMsg())
+ .build();
}
/**
* Returns a CommandStatus from the protocol buffers.
@@ -95,20 +107,36 @@ public class CommandStatus {
/**
* Builder class for CommandStatus.
*/
- public static final class CommandStatusBuilder {
+ public static class CommandStatusBuilder {
private SCMCommandProto.Type type;
private Long cmdId;
private StorageContainerDatanodeProtocolProtos.CommandStatus.Status status;
private String msg;
- private CommandStatusBuilder() {
+ CommandStatusBuilder() {
}
public static CommandStatusBuilder newBuilder() {
return new CommandStatusBuilder();
}
+ public Type getType() {
+ return type;
+ }
+
+ public Long getCmdId() {
+ return cmdId;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
public CommandStatusBuilder setType(Type commandType) {
this.type = commandType;
return this;
@@ -130,12 +158,7 @@ public class CommandStatus {
}
public CommandStatus build() {
- CommandStatus commandStatus = new CommandStatus();
- commandStatus.type = this.type;
- commandStatus.msg = this.msg;
- commandStatus.status = this.status;
- commandStatus.cmdId = this.cmdId;
- return commandStatus;
+ return new CommandStatus(type, cmdId, status, msg);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlockCommandStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlockCommandStatus.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlockCommandStatus.java
new file mode 100644
index 0000000..2659ab3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlockCommandStatus.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
+
+public class DeleteBlockCommandStatus extends CommandStatus {
+
+ private ContainerBlocksDeletionACKProto blocksDeletionAck = null;
+
+ public DeleteBlockCommandStatus(Type type, Long cmdId,
+ StorageContainerDatanodeProtocolProtos.CommandStatus.Status status,
+ String msg, ContainerBlocksDeletionACKProto blocksDeletionAck) {
+ super(type, cmdId, status, msg);
+ this.blocksDeletionAck = blocksDeletionAck;
+ }
+
+ public void setBlocksDeletionAck(
+ ContainerBlocksDeletionACKProto deletionAck) {
+ blocksDeletionAck = deletionAck;
+ }
+
+ @Override
+ public CommandStatus getFromProtoBuf(
+ StorageContainerDatanodeProtocolProtos.CommandStatus cmdStatusProto) {
+ return DeleteBlockCommandStatusBuilder.newBuilder()
+ .setBlockDeletionAck(cmdStatusProto.getBlockDeletionAck())
+ .setCmdId(cmdStatusProto.getCmdId())
+ .setStatus(cmdStatusProto.getStatus())
+ .setType(cmdStatusProto.getType())
+ .setMsg(cmdStatusProto.getMsg())
+ .build();
+ }
+
+ @Override
+ public StorageContainerDatanodeProtocolProtos.CommandStatus getProtoBufMessage() {
+ StorageContainerDatanodeProtocolProtos.CommandStatus.Builder builder =
+ StorageContainerDatanodeProtocolProtos.CommandStatus.newBuilder()
+ .setCmdId(this.getCmdId())
+ .setStatus(this.getStatus())
+ .setType(this.getType());
+ if (blocksDeletionAck != null) {
+ builder.setBlockDeletionAck(blocksDeletionAck);
+ }
+ if (this.getMsg() != null) {
+ builder.setMsg(this.getMsg());
+ }
+ return builder.build();
+ }
+
+ public static final class DeleteBlockCommandStatusBuilder
+ extends CommandStatusBuilder {
+ private ContainerBlocksDeletionACKProto blocksDeletionAck = null;
+
+ public static DeleteBlockCommandStatusBuilder newBuilder() {
+ return new DeleteBlockCommandStatusBuilder();
+ }
+
+ public DeleteBlockCommandStatusBuilder setBlockDeletionAck(
+ ContainerBlocksDeletionACKProto deletionAck) {
+ this.blocksDeletionAck = deletionAck;
+ return this;
+ }
+
+ @Override
+ public CommandStatus build() {
+ return new DeleteBlockCommandStatus(getType(), getCmdId(), getStatus(),
+ getMsg(), blocksDeletionAck);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
index b9cf6f9..4e1e27e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
@@ -23,11 +23,6 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos
- .ContainerBlocksDeletionACKResponseProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
@@ -169,16 +164,4 @@ public class StorageContainerDatanodeProtocolClientSideTranslatorPB
return response;
}
- @Override
- public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- ContainerBlocksDeletionACKProto deletedBlocks) throws IOException {
- final ContainerBlocksDeletionACKResponseProto resp;
- try {
- resp = rpcProxy.sendContainerBlocksDeletionACK(NULL_RPC_CONTROLLER,
- deletedBlocks);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- return resp;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
index ed01822..8622332 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
@@ -33,11 +33,6 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos
- .ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
@@ -96,15 +91,4 @@ public class StorageContainerDatanodeProtocolServerSideTranslatorPB
}
}
-
- @Override
- public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- RpcController controller, ContainerBlocksDeletionACKProto request)
- throws ServiceException {
- try {
- return impl.sendContainerBlocksDeletionACK(request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 78758cb..982029c 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -80,7 +80,7 @@ message SCMHeartbeatRequestProto {
required DatanodeDetailsProto datanodeDetails = 1;
optional NodeReportProto nodeReport = 2;
optional ContainerReportsProto containerReport = 3;
- optional CommandStatusReportsProto commandStatusReport = 4;
+ repeated CommandStatusReportsProto commandStatusReports = 4;
optional ContainerActionsProto containerActions = 5;
optional PipelineActionsProto pipelineActions = 6;
optional PipelineReportsProto pipelineReports = 7;
@@ -145,6 +145,7 @@ message CommandStatus {
required Status status = 2 [default = PENDING];
required SCMCommandProto.Type type = 3;
optional string msg = 4;
+ optional ContainerBlocksDeletionACKProto blockDeletionAck = 5;
}
message ContainerActionsProto {
@@ -272,10 +273,6 @@ message ContainerBlocksDeletionACKProto {
required string dnId = 2;
}
-// SendACK response returned by datanode to SCM, currently empty.
-message ContainerBlocksDeletionACKResponseProto {
-}
-
/**
This command asks the datanode to close a specific container.
*/
@@ -386,8 +383,4 @@ service StorageContainerDatanodeProtocolService {
*/
rpc sendHeartbeat (SCMHeartbeatRequestProto) returns (SCMHeartbeatResponseProto);
- /**
- * Sends the block deletion ACK to SCM.
- */
- rpc sendContainerBlocksDeletionACK (ContainerBlocksDeletionACKProto) returns (ContainerBlocksDeletionACKResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
index 27b6272..3e45596 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatus;
import org.apache.hadoop.hdds.scm.VersionInfo;
@@ -30,11 +32,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos
- .ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
@@ -196,10 +193,12 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
sendHeartbeat(SCMHeartbeatRequestProto heartbeat) throws IOException {
rpcCount.incrementAndGet();
heartbeatCount.incrementAndGet();
- if(heartbeat.hasCommandStatusReport()){
- cmdStatusList.addAll(heartbeat.getCommandStatusReport()
- .getCmdStatusList());
- commandStatusReport.incrementAndGet();
+ if (heartbeat.getCommandStatusReportsCount() != 0) {
+ for (CommandStatusReportsProto statusReport : heartbeat
+ .getCommandStatusReportsList()) {
+ cmdStatusList.addAll(statusReport.getCmdStatusList());
+ commandStatusReport.incrementAndGet();
+ }
}
sleepIfNeeded();
return SCMHeartbeatResponseProto.newBuilder().addAllCommands(
@@ -305,13 +304,6 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
return 0;
}
- @Override
- public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- ContainerBlocksDeletionACKProto request) throws IOException {
- return ContainerBlocksDeletionACKResponseProto
- .newBuilder().getDefaultInstanceForType();
- }
-
/**
* Reset the mock Scm for test to get a fresh start without rebuild MockScm.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
index 811599f..1e82326 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java
@@ -138,9 +138,7 @@ public class TestReportPublisher {
new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat("Unit test ReportManager Thread - %d").build());
publisher.init(dummyContext, executorService);
- Assert.assertEquals(0,
- ((CommandStatusReportPublisher) publisher).getReport()
- .getCmdStatusCount());
+ Assert.assertNull(((CommandStatusReportPublisher) publisher).getReport());
// Insert to status object to state context map and then get the report.
CommandStatus obj1 = CommandStatus.CommandStatusBuilder.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
index 69a6a33..606940b 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/TestHeartbeatEndpointTask.java
@@ -77,7 +77,7 @@ public class TestHeartbeatEndpointTask {
Assert.assertTrue(heartbeat.hasDatanodeDetails());
Assert.assertFalse(heartbeat.hasNodeReport());
Assert.assertFalse(heartbeat.hasContainerReport());
- Assert.assertFalse(heartbeat.hasCommandStatusReport());
+ Assert.assertTrue(heartbeat.getCommandStatusReportsCount() == 0);
Assert.assertFalse(heartbeat.hasContainerActions());
}
@@ -108,7 +108,7 @@ public class TestHeartbeatEndpointTask {
Assert.assertTrue(heartbeat.hasDatanodeDetails());
Assert.assertTrue(heartbeat.hasNodeReport());
Assert.assertFalse(heartbeat.hasContainerReport());
- Assert.assertFalse(heartbeat.hasCommandStatusReport());
+ Assert.assertTrue(heartbeat.getCommandStatusReportsCount() == 0);
Assert.assertFalse(heartbeat.hasContainerActions());
}
@@ -139,7 +139,7 @@ public class TestHeartbeatEndpointTask {
Assert.assertTrue(heartbeat.hasDatanodeDetails());
Assert.assertFalse(heartbeat.hasNodeReport());
Assert.assertTrue(heartbeat.hasContainerReport());
- Assert.assertFalse(heartbeat.hasCommandStatusReport());
+ Assert.assertTrue(heartbeat.getCommandStatusReportsCount() == 0);
Assert.assertFalse(heartbeat.hasContainerActions());
}
@@ -170,7 +170,7 @@ public class TestHeartbeatEndpointTask {
Assert.assertTrue(heartbeat.hasDatanodeDetails());
Assert.assertFalse(heartbeat.hasNodeReport());
Assert.assertFalse(heartbeat.hasContainerReport());
- Assert.assertTrue(heartbeat.hasCommandStatusReport());
+ Assert.assertTrue(heartbeat.getCommandStatusReportsCount() != 0);
Assert.assertFalse(heartbeat.hasContainerActions());
}
@@ -201,7 +201,7 @@ public class TestHeartbeatEndpointTask {
Assert.assertTrue(heartbeat.hasDatanodeDetails());
Assert.assertFalse(heartbeat.hasNodeReport());
Assert.assertFalse(heartbeat.hasContainerReport());
- Assert.assertFalse(heartbeat.hasCommandStatusReport());
+ Assert.assertTrue(heartbeat.getCommandStatusReportsCount() == 0);
Assert.assertTrue(heartbeat.hasContainerActions());
}
@@ -235,7 +235,7 @@ public class TestHeartbeatEndpointTask {
Assert.assertTrue(heartbeat.hasDatanodeDetails());
Assert.assertTrue(heartbeat.hasNodeReport());
Assert.assertTrue(heartbeat.hasContainerReport());
- Assert.assertTrue(heartbeat.hasCommandStatusReport());
+ Assert.assertTrue(heartbeat.getCommandStatusReportsCount() != 0);
Assert.assertTrue(heartbeat.hasContainerActions());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java
index 25420fe..8702a42 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java
@@ -61,7 +61,8 @@ public class DatanodeDeletedBlockTransactions {
try {
ContainerWithPipeline containerWithPipeline =
mappingService.getContainerWithPipeline(tx.getContainerID());
- if (containerWithPipeline.getContainerInfo().isContainerOpen()) {
+ if (containerWithPipeline.getContainerInfo().isContainerOpen()
+ || containerWithPipeline.getPipeline().isEmpty()) {
return false;
}
pipeline = containerWithPipeline.getPipeline();
@@ -70,25 +71,19 @@ public class DatanodeDeletedBlockTransactions {
return false;
}
- if (pipeline == null) {
- SCMBlockDeletingService.LOG.warn(
- "Container {} not found, continue to process next",
- tx.getContainerID());
- return false;
- }
-
+ boolean success = false;
for (DatanodeDetails dd : pipeline.getMachines()) {
UUID dnID = dd.getUuid();
if (dnsWithTransactionCommitted == null ||
!dnsWithTransactionCommitted.contains(dnID)) {
// Transaction need not be sent to dns which have already committed it
- addTransactionToDN(dnID, tx);
+ success = addTransactionToDN(dnID, tx);
}
}
- return true;
+ return success;
}
- private void addTransactionToDN(UUID dnID, DeletedBlocksTransaction tx) {
+ private boolean addTransactionToDN(UUID dnID, DeletedBlocksTransaction tx) {
if (transactions.containsKey(dnID)) {
List<DeletedBlocksTransaction> txs = transactions.get(dnID);
if (txs != null && txs.size() < maximumAllowedTXNum) {
@@ -103,14 +98,17 @@ public class DatanodeDeletedBlockTransactions {
if (!hasContained) {
txs.add(tx);
currentTXNum++;
+ return true;
}
}
} else {
currentTXNum++;
transactions.put(dnID, tx);
+ return true;
}
SCMBlockDeletingService.LOG
.debug("Transaction added: {} <- TX({})", dnID, tx.getTxID());
+ return false;
}
Set<UUID> getDatanodeIDs() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index 49af65c..68435d1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -23,10 +23,16 @@ import com.google.common.primitives.Longs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto
.DeleteBlockTransactionResult;
+import org.apache.hadoop.hdds.scm.command
+ .CommandStatusReportHandler.DeleteBlockStatus;
import org.apache.hadoop.hdds.scm.container.Mapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
@@ -75,9 +81,10 @@ import static org.apache.hadoop.ozone.OzoneConsts.DELETED_BLOCK_DB;
* equally same chance to be retrieved which only depends on the nature
* order of the transaction ID.
*/
-public class DeletedBlockLogImpl implements DeletedBlockLog {
+public class DeletedBlockLogImpl
+ implements DeletedBlockLog, EventHandler<DeleteBlockStatus> {
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(DeletedBlockLogImpl.class);
private static final byte[] LATEST_TXID =
@@ -123,7 +130,7 @@ public class DeletedBlockLogImpl implements DeletedBlockLog {
}
@VisibleForTesting
- MetadataStore getDeletedStore() {
+ public MetadataStore getDeletedStore() {
return deletedStore;
}
@@ -269,6 +276,8 @@ public class DeletedBlockLogImpl implements DeletedBlockLog {
deletedStore.delete(Longs.toByteArray(txID));
}
}
+ LOG.debug("Datanode txId={} containerId={} committed by dnId={}",
+ txID, containerId, dnID);
} catch (IOException e) {
LOG.warn("Could not commit delete block transaction: " +
transactionResult.getTxID(), e);
@@ -407,4 +416,13 @@ public class DeletedBlockLogImpl implements DeletedBlockLog {
lock.unlock();
}
}
+
+ @Override
+ public void onMessage(DeleteBlockStatus deleteBlockStatus,
+ EventPublisher publisher) {
+ ContainerBlocksDeletionACKProto ackProto =
+ deleteBlockStatus.getCmdStatus().getBlockDeletionAck();
+ commitTransactions(ackProto.getResultsList(),
+ UUID.fromString(ackProto.getDnId()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
index de3fe26..b85d77f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java
@@ -165,7 +165,7 @@ public class SCMBlockDeletingService extends BackgroundService {
// We should stop caching new commands if num of un-processed
// command is bigger than a limit, e.g 50. In case datanode goes
// offline for sometime, the cached commands be flooded.
- eventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND,
+ eventPublisher.fireEvent(SCMEvents.RETRIABLE_DATANODE_COMMAND,
new CommandForDatanode<>(dnId, new DeleteBlocksCommand(dnTXs)));
LOG.debug(
"Added delete block command for datanode {} in the queue,"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
index 054665a..c0de382 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/command/CommandStatusReportHandler.java
@@ -63,8 +63,10 @@ public class CommandStatusReportHandler implements
CloseContainerStatus(cmdStatus));
break;
case deleteBlocksCommand:
- publisher.fireEvent(SCMEvents.DELETE_BLOCK_STATUS, new
- DeleteBlockCommandStatus(cmdStatus));
+ if (cmdStatus.getStatus() == CommandStatus.Status.EXECUTED) {
+ publisher.fireEvent(SCMEvents.DELETE_BLOCK_STATUS,
+ new DeleteBlockStatus(cmdStatus));
+ }
break;
default:
LOGGER.debug("CommandStatus of type:{} not handled in " +
@@ -120,8 +122,8 @@ public class CommandStatusReportHandler implements
/**
* Wrapper event for DeleteBlock Command.
*/
- public static class DeleteBlockCommandStatus extends CommandStatusEvent {
- public DeleteBlockCommandStatus(CommandStatus cmdStatus) {
+ public static class DeleteBlockStatus extends CommandStatusEvent {
+ public DeleteBlockStatus(CommandStatus cmdStatus) {
super(cmdStatus);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index eb0a0b4..71e17e9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -176,7 +175,10 @@ public class ContainerMapping implements Mapping {
}
/**
- * Returns the ContainerInfo from the container ID.
+ * Returns the ContainerInfo and pipeline from the containerID. If container
+ * has no available replicas in datanodes it returns pipeline with no
+ * datanodes and empty leaderID . Pipeline#isEmpty can be used to check for
+ * an empty pipeline.
*
* @param containerID - ID of container.
* @return - ContainerWithPipeline such as creation state and the pipeline.
@@ -200,6 +202,7 @@ public class ContainerMapping implements Mapping {
contInfo = ContainerInfo.fromProtobuf(temp);
Pipeline pipeline;
+ String leaderId = "";
if (contInfo.isContainerOpen()) {
// If pipeline with given pipeline Id already exist return it
pipeline = pipelineSelector.getPipeline(contInfo.getPipelineID());
@@ -207,14 +210,12 @@ public class ContainerMapping implements Mapping {
// For close containers create pipeline from datanodes with replicas
Set<DatanodeDetails> dnWithReplicas = containerStateManager
.getContainerReplicas(contInfo.containerID());
- if (dnWithReplicas.size() == 0) {
- throw new SCMException("Can't create a pipeline for container with "
- + "no replica.", ResultCodes.NO_REPLICA_FOUND);
+ if (!dnWithReplicas.isEmpty()) {
+ leaderId = dnWithReplicas.iterator().next().getUuidString();
}
- pipeline =
- new Pipeline(dnWithReplicas.iterator().next().getUuidString(),
- contInfo.getState(), ReplicationType.STAND_ALONE,
- contInfo.getReplicationFactor(), PipelineID.randomId());
+ pipeline = new Pipeline(leaderId, contInfo.getState(),
+ ReplicationType.STAND_ALONE, contInfo.getReplicationFactor(),
+ PipelineID.randomId());
dnWithReplicas.forEach(pipeline::addMember);
}
return new ContainerWithPipeline(contInfo, pipeline);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 745e052..77b8713 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -23,8 +23,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
.CloseContainerStatus;
-import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
- .DeleteBlockCommandStatus;
+import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
.ReplicationStatus;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler
@@ -128,6 +127,10 @@ public final class SCMEvents {
public static final Event<CommandForDatanode> DATANODE_COMMAND =
new TypedEvent<>(CommandForDatanode.class, "Datanode_Command");
+ public static final TypedEvent<CommandForDatanode>
+ RETRIABLE_DATANODE_COMMAND =
+ new TypedEvent<>(CommandForDatanode.class, "Retriable_Datanode_Command");
+
/**
* A Close Container Event can be triggered under many condition. Some of them
* are: 1. A Container is full, then we stop writing further information to
@@ -179,7 +182,7 @@ public final class SCMEvents {
* status for Replication SCMCommand is received.
*/
public static final Event<ReplicationStatus> REPLICATION_STATUS = new
- TypedEvent<>(ReplicationStatus.class, "ReplicateCommandStatus");
+ TypedEvent<>(ReplicationStatus.class, "Replicate_Command_Status");
/**
* This event will be triggered by CommandStatusReportHandler whenever a
* status for CloseContainer SCMCommand is received.
@@ -187,15 +190,15 @@ public final class SCMEvents {
public static final Event<CloseContainerStatus>
CLOSE_CONTAINER_STATUS =
new TypedEvent<>(CloseContainerStatus.class,
- "CloseContainerCommandStatus");
+ "Close_Container_Command_Status");
/**
* This event will be triggered by CommandStatusReportHandler whenever a
* status for DeleteBlock SCMCommand is received.
*/
- public static final Event<DeleteBlockCommandStatus>
+ public static final TypedEvent<CommandStatusReportHandler.DeleteBlockStatus>
DELETE_BLOCK_STATUS =
- new TypedEvent<>(DeleteBlockCommandStatus.class,
- "DeleteBlockCommandStatus");
+ new TypedEvent<>(CommandStatusReportHandler.DeleteBlockStatus.class,
+ "Delete_Block_Status");
/**
* This event will be triggered while processing container reports from DN
@@ -203,7 +206,7 @@ public final class SCMEvents {
* deleteTransactionID on SCM.
*/
public static final Event<PendingDeleteStatusList> PENDING_DELETE_STATUS =
- new TypedEvent<>(PendingDeleteStatusList.class, "PendingDeleteStatus");
+ new TypedEvent<>(PendingDeleteStatusList.class, "Pending_Delete_Status");
/**
* This is the command for ReplicationManager to handle under/over
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index e65de8b..d9a0875 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -121,10 +121,13 @@ public final class SCMDatanodeHeartbeatDispatcher {
heartbeat.getPipelineActions()));
}
- if (heartbeat.hasCommandStatusReport()) {
- eventPublisher.fireEvent(CMD_STATUS_REPORT,
- new CommandStatusReportFromDatanode(datanodeDetails,
- heartbeat.getCommandStatusReport()));
+ if (heartbeat.getCommandStatusReportsCount() != 0) {
+ for (CommandStatusReportsProto commandStatusReport : heartbeat
+ .getCommandStatusReportsList()) {
+ eventPublisher.fireEvent(CMD_STATUS_REPORT,
+ new CommandStatusReportFromDatanode(datanodeDetails,
+ commandStatusReport));
+ }
}
return commands;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 4a0d3e5..9c6fa88 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -49,15 +49,6 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos
- .ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos
- .ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult;
-
import static org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto
@@ -100,7 +91,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.LinkedList;
import java.util.List;
-import java.util.UUID;
import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
@@ -242,18 +232,6 @@ public class SCMDatanodeProtocolServer implements
.addAllCommands(cmdResponses).build();
}
- @Override
- public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- ContainerBlocksDeletionACKProto acks) throws IOException {
- if (acks.getResultsCount() > 0) {
- List<DeleteBlockTransactionResult> resultList = acks.getResultsList();
- scm.getScmBlockManager().getDeletedBlockLog()
- .commitTransactions(resultList, UUID.fromString(acks.getDnId()));
- }
- return ContainerBlocksDeletionACKResponseProto.newBuilder()
- .getDefaultInstanceForType();
- }
-
/**
* Returns a SCMCommandRepose from the SCM Command.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 52af62d..bb72075 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
+import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
import org.apache.hadoop.hdds.scm.block.PendingDeleteHandler;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler;
import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler;
@@ -68,6 +69,7 @@ import org.apache.hadoop.hdds.scm.pipelines.PipelineReportHandler;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RPC;
@@ -253,6 +255,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
commandWatcherLeaseManager = new LeaseManager<>("CommandWatcher",
watcherTimeout);
+ RetriableDatanodeEventWatcher retriableDatanodeEventWatcher =
+ new RetriableDatanodeEventWatcher<>(
+ SCMEvents.RETRIABLE_DATANODE_COMMAND,
+ SCMEvents.DELETE_BLOCK_STATUS,
+ commandWatcherLeaseManager);
+ retriableDatanodeEventWatcher.start(eventQueue);
+
//TODO: support configurable containerPlacement policy
ContainerPlacementPolicy containerPlacementPolicy =
new SCMContainerPlacementCapacity(scmNodeManager, conf);
@@ -282,6 +291,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
httpServer = new StorageContainerManagerHttpServer(conf);
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
+ eventQueue.addHandler(SCMEvents.RETRIABLE_DATANODE_COMMAND, scmNodeManager);
eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
eventQueue.addHandler(SCMEvents.CONTAINER_REPORT, containerReportHandler);
eventQueue.addHandler(SCMEvents.CONTAINER_ACTIONS, actionsHandler);
@@ -296,6 +306,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
replicationStatus.getChillModeStatusListener());
eventQueue
.addHandler(SCMEvents.PENDING_DELETE_STATUS, pendingDeleteHandler);
+ eventQueue.addHandler(SCMEvents.DELETE_BLOCK_STATUS,
+ (DeletedBlockLogImpl) scmBlockManager.getDeletedBlockLog());
eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS,
pipelineActionEventHandler);
eventQueue.addHandler(SCMEvents.PIPELINE_CLOSE, pipelineCloseHandler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/RetriableDatanodeEventWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/RetriableDatanodeEventWatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/RetriableDatanodeEventWatcher.java
new file mode 100644
index 0000000..2a50bca
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/RetriableDatanodeEventWatcher.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler.CommandStatusEvent;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.EventWatcher;
+import org.apache.hadoop.ozone.lease.LeaseManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * EventWatcher for start events and completion events with payload of type
+ * RetriablePayload and RetriableCompletionPayload respectively.
+ */
+public class RetriableDatanodeEventWatcher<T extends CommandStatusEvent>
+ extends EventWatcher<CommandForDatanode, T> {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(RetriableDatanodeEventWatcher.class);
+
+ public RetriableDatanodeEventWatcher(Event<CommandForDatanode> startEvent,
+ Event<T> completionEvent, LeaseManager<Long> leaseManager) {
+ super(startEvent, completionEvent, leaseManager);
+ }
+
+ @Override
+ protected void onTimeout(EventPublisher publisher,
+ CommandForDatanode payload) {
+ LOG.info("RetriableDatanodeCommand type={} with id={} timed out. Retrying.",
+ payload.getCommand().getType(), payload.getId());
+ //put back to the original queue
+ publisher.fireEvent(SCMEvents.RETRIABLE_DATANODE_COMMAND, payload);
+ }
+
+ @Override
+ protected void onFinished(EventPublisher publisher,
+ CommandForDatanode payload) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
new file mode 100644
index 0000000..b1d2838
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
index 65a2e29..afa25e2 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
@@ -67,20 +67,20 @@ public class TestCommandStatusReportHandler implements EventPublisher {
CommandStatusReportFromDatanode report = this.getStatusReport(Collections
.emptyList());
cmdStatusReportHandler.onMessage(report, this);
- assertFalse(logCapturer.getOutput().contains("DeleteBlockCommandStatus"));
+ assertFalse(logCapturer.getOutput().contains("Delete_Block_Status"));
assertFalse(logCapturer.getOutput().contains(
- "CloseContainerCommandStatus"));
- assertFalse(logCapturer.getOutput().contains("ReplicateCommandStatus"));
+ "Close_Container_Command_Status"));
+ assertFalse(logCapturer.getOutput().contains("Replicate_Command_Status"));
report = this.getStatusReport(this.getCommandStatusList());
cmdStatusReportHandler.onMessage(report, this);
assertTrue(logCapturer.getOutput().contains("firing event of type " +
- "DeleteBlockCommandStatus"));
+ "Delete_Block_Status"));
assertTrue(logCapturer.getOutput().contains("firing event of type " +
- "CloseContainerCommandStatus"));
+ "Close_Container_Command_Status"));
assertTrue(logCapturer.getOutput().contains("firing event of type " +
- "ReplicateCommandStatus"));
+ "Replicate_Command_Status"));
assertTrue(logCapturer.getOutput().contains("type: " +
"closeContainerCommand"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
index 6a0b909..f3cd4ea 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
@@ -127,7 +127,7 @@ public class TestSCMDatanodeHeartbeatDispatcher {
SCMHeartbeatRequestProto.newBuilder()
.setDatanodeDetails(datanodeDetails.getProtoBufMessage())
.setContainerReport(containerReport)
- .setCommandStatusReport(commandStatusReport)
+ .addCommandStatusReports(commandStatusReport)
.build();
dispatcher.dispatch(heartbeat);
Assert.assertEquals(2, eventReceived.get());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7ff8c05/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index fe53bcc..ac3ad5d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.junit.Assert.fail;
@@ -188,6 +189,8 @@ public class TestStorageContainerManager {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 100,
TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 100,
+ TimeUnit.MILLISECONDS);
conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
3000,
TimeUnit.MILLISECONDS);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDDS-557. DeadNodeHandler should
handle exception from removeContainerHandler api. Contributed by Ajay Kumar.
Posted by in...@apache.org.
HDDS-557. DeadNodeHandler should handle exception from removeContainerHandler api. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc80ac23
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc80ac23
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc80ac23
Branch: refs/heads/HDFS-13532
Commit: cc80ac23156e1e91c1f77df65fa53504fbb34141
Parents: f7ff8c0
Author: Ajay Kumar <aj...@apache.com>
Authored: Mon Oct 1 11:21:26 2018 -0700
Committer: Ajay Kumar <aj...@apache.com>
Committed: Mon Oct 1 11:21:26 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 16 ++++++++++++--
.../hdds/scm/node/TestDeadNodeHandler.java | 23 ++++++++++++++++++++
2 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc80ac23/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
index 88f5df7..253b3ec 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
@@ -72,8 +72,13 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
datanodeDetails.getUuid());
for (ContainerID container : containers) {
try {
- containerStateManager.removeContainerReplica(container,
- datanodeDetails);
+ try {
+ containerStateManager.removeContainerReplica(container,
+ datanodeDetails);
+ } catch (SCMException ex) {
+ LOG.info("DataNode {} doesn't have replica for container {}.",
+ datanodeDetails.getUuid(), container.getId());
+ }
if (!containerStateManager.isOpen(container)) {
ReplicationRequest replicationRequest =
@@ -90,4 +95,11 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
}
}
}
+
+ /**
+ * Returns logger.
+ * */
+ public static Logger getLogger() {
+ return LOG;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc80ac23/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 698332e..6966322 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -196,6 +196,29 @@ public class TestDeadNodeHandler {
Assert.assertTrue(nodeStat.get().getScmUsed().get() == 0);
}
+ @Test
+ public void testOnMessageReplicaFailure() throws Exception {
+ DatanodeDetails dn1 = TestUtils.randomDatanodeDetails();
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(DeadNodeHandler.getLogger());
+ String storagePath1 = GenericTestUtils.getRandomizedTempPath()
+ .concat("/" + dn1.getUuidString());
+
+ StorageReportProto storageOne = TestUtils.createStorageReport(
+ dn1.getUuid(), storagePath1, 100, 10, 90, null);
+ nodeReportHandler.onMessage(getNodeReport(dn1, storageOne),
+ Mockito.mock(EventPublisher.class));
+
+ ContainerInfo container1 =
+ TestUtils.allocateContainer(containerStateManager);
+ registerReplicas(node2ContainerMap, dn1, container1);
+
+ deadNodeHandler.onMessage(dn1, eventQueue);
+ Assert.assertTrue(logCapturer.getOutput().contains(
+ "DataNode " + dn1.getUuid() + " doesn't have replica for container "
+ + container1.getContainerID()));
+ }
+
private void registerReplicas(ContainerStateManager csm,
ContainerInfo container, DatanodeDetails... datanodes) {
csm.getContainerStateMap()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-8665. Added Yarn service
cancel upgrade option. Contributed by Chandni Singh
Posted by in...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
index 3c856ec..153ab46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.service.ServiceContext;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.component.ComponentEvent;
@@ -113,7 +112,8 @@ public class ContainerLaunchService extends AbstractService{
.startContainerAsync(container,
launcher.completeContainerLaunch());
} else {
- LOG.info("reInitializing container {}", container.getId());
+ LOG.info("reInitializing container {} with version {}",
+ container.getId(), componentLaunchContext.getServiceVersion());
instance.getComponent().getScheduler().getNmClient()
.reInitializeContainerAsync(container.getId(),
launcher.completeContainerLaunch(), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
index 49ecd2e..6f37967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
@@ -30,6 +30,8 @@ import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
@@ -141,4 +143,15 @@ public class ClientAMProtocolPBClientImpl
}
return null;
}
+
+ @Override
+ public CancelUpgradeResponseProto cancelUpgrade(
+ CancelUpgradeRequestProto request) throws IOException, YarnException {
+ try {
+ return proxy.cancelUpgrade(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
index eab3f9f..071c357 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.service.impl.pb.service;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
@@ -116,4 +118,15 @@ public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
throw new ServiceException(e);
}
}
+
+ @Override
+ public CancelUpgradeResponseProto cancelUpgrade(
+ RpcController controller, CancelUpgradeRequestProto request)
+ throws ServiceException {
+ try {
+ return real.cancelUpgrade(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index ac90992..c12c340 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -167,9 +167,8 @@ public class ProviderUtils implements YarnServiceConstants {
public static Path initCompInstanceDir(SliderFileSystem fs,
ContainerLaunchService.ComponentLaunchContext compLaunchContext,
ComponentInstance instance) {
- Path compDir = new Path(new Path(fs.getAppDir(), "components"),
- compLaunchContext.getServiceVersion() + "/" +
- compLaunchContext.getName());
+ Path compDir = fs.getComponentDir(compLaunchContext.getServiceVersion(),
+ compLaunchContext.getName());
Path compInstanceDir = new Path(compDir, instance.getCompInstanceName());
instance.setCompInstanceDir(compInstanceDir);
return compInstanceDir;
@@ -184,7 +183,9 @@ public class ProviderUtils implements YarnServiceConstants {
ServiceContext context) throws IOException {
Path compInstanceDir = initCompInstanceDir(fs, compLaunchContext, instance);
if (!fs.getFileSystem().exists(compInstanceDir)) {
- log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
+ log.info("{} version {} : Creating dir on hdfs: {}",
+ instance.getCompInstanceId(), compLaunchContext.getServiceVersion(),
+ compInstanceDir);
fs.getFileSystem().mkdirs(compInstanceDir,
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index b588e88..0eb54ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -603,7 +603,7 @@ public class ServiceApiUtil {
public static void validateInstancesUpgrade(List<Container>
liveContainers) throws YarnException {
for (Container liveContainer : liveContainers) {
- if (!liveContainer.getState().equals(ContainerState.NEEDS_UPGRADE)) {
+ if (!isUpgradable(liveContainer)) {
// Nothing to upgrade
throw new YarnException(String.format(
ERROR_COMP_INSTANCE_DOES_NOT_NEED_UPGRADE,
@@ -613,6 +613,16 @@ public class ServiceApiUtil {
}
/**
+ * Returns whether the container can be upgraded in the current state.
+ */
+ public static boolean isUpgradable(Container container) {
+
+ return container.getState() != null &&
+ (container.getState().equals(ContainerState.NEEDS_UPGRADE) ||
+ container.getState().equals(ContainerState.FAILED_UPGRADE));
+ }
+
+ /**
* Validates the components that are requested to upgrade require an upgrade.
* It returns the instances of the components which need upgrade.
*/
@@ -629,7 +639,7 @@ public class ServiceApiUtil {
ERROR_COMP_DOES_NOT_NEED_UPGRADE, liveComp.getName()));
}
liveComp.getContainers().forEach(liveContainer -> {
- if (liveContainer.getState().equals(ContainerState.NEEDS_UPGRADE)) {
+ if (isUpgradable(liveContainer)) {
containerNeedUpgrade.add(liveContainer);
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
index d6d664e..c776476 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.service.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
@@ -48,4 +50,51 @@ public class SliderFileSystem extends CoreFileSystem {
public Path getAppDir() {
return this.appDir;
}
+
+ /**
+ * Returns the component directory path.
+ *
+ * @param serviceVersion service version
+ * @param compName component name
+ * @return component directory
+ */
+ public Path getComponentDir(String serviceVersion, String compName) {
+ return new Path(new Path(getAppDir(), "components"),
+ serviceVersion + "/" + compName);
+ }
+
+ /**
+ * Deletes the component directory.
+ *
+ * @param serviceVersion
+ * @param compName
+ * @throws IOException
+ */
+ public void deleteComponentDir(String serviceVersion, String compName)
+ throws IOException {
+ Path path = getComponentDir(serviceVersion, compName);
+ if (fileSystem.exists(path)) {
+ fileSystem.delete(path, true);
+ LOG.debug("deleted dir {}", path);
+ }
+ }
+
+ /**
+ * Deletes the components version directory.
+ *
+ * @param serviceVersion
+ * @throws IOException
+ */
+ public void deleteComponentsVersionDirIfEmpty(String serviceVersion)
+ throws IOException {
+ Path path = new Path(new Path(getAppDir(), "components"), serviceVersion);
+ if (fileSystem.exists(path) && fileSystem.listStatus(path).length == 0) {
+ fileSystem.delete(path, true);
+ LOG.info("deleted dir {}", path);
+ }
+ }
+
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ SliderFileSystem.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
index 169f765..bcf893e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
@@ -28,6 +28,8 @@ service ClientAMProtocolService {
rpc stop(StopRequestProto) returns (StopResponseProto);
rpc upgradeService(UpgradeServiceRequestProto)
returns (UpgradeServiceResponseProto);
+ rpc cancelUpgrade(CancelUpgradeRequestProto)
+ returns (CancelUpgradeResponseProto);
rpc restartService(RestartServiceRequestProto)
returns (RestartServiceResponseProto);
rpc upgrade(CompInstancesUpgradeRequestProto) returns
@@ -73,6 +75,12 @@ message UpgradeServiceResponseProto {
optional string error = 1;
}
+message CancelUpgradeRequestProto {
+}
+
+message CancelUpgradeResponseProto {
+}
+
message RestartServiceRequestProto {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockRunningServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockRunningServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockRunningServiceContext.java
index 321b2cd..b685f4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockRunningServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockRunningServiceContext.java
@@ -94,21 +94,25 @@ public class MockRunningServiceContext extends ServiceContext {
return mockLaunchService;
}
- @Override public ServiceUtils.ProcessTerminationHandler
- getTerminationHandler() {
+ @Override
+ public ServiceUtils.ProcessTerminationHandler getTerminationHandler() {
return new
- ServiceUtils.ProcessTerminationHandler() {
- public void terminate(int exitCode) {
- }
- };
+ ServiceUtils.ProcessTerminationHandler() {
+ public void terminate(int exitCode) {
+ }
+ };
+ }
+
+ @Override
+ protected ServiceManager createServiceManager() {
+ return ServiceTestUtils.createServiceManager(
+ MockRunningServiceContext.this);
}
};
this.scheduler.init(fsWatcher.getConf());
- ServiceTestUtils.createServiceManager(this);
-
doNothing().when(mockLaunchService).
reInitCompInstance(anyObject(), anyObject(), anyObject(), anyObject());
stabilizeComponents(this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 6b49ab0..58db752 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -384,6 +384,7 @@ public class ServiceTestUtils {
conf.set(YARN_SERVICE_BASE_PATH, serviceBasePath.toString());
try {
fs = new SliderFileSystem(conf);
+ fs.setAppDir(new Path(serviceBasePath.toString()));
} catch (IOException e) {
Throwables.propagate(e);
}
@@ -532,7 +533,6 @@ public class ServiceTestUtils {
GenericTestUtils.waitFor(() -> {
try {
Service retrievedApp = client.getStatus(exampleApp.getName());
- System.out.println(retrievedApp);
return retrievedApp.getState() == desiredState;
} catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java
index a37cabe..406eea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.service.api.records.ComponentState;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
@@ -65,7 +66,7 @@ public class TestServiceManager {
initUpgrade(context, "v2", false, false, false);
ServiceManager manager = context.getServiceManager();
//make components stable by upgrading all instances
- upgradeAllInstances(context);
+ upgradeAndReadyAllInstances(context);
context.scheduler.getDispatcher().getEventHandler().handle(
new ServiceEvent(ServiceEventType.START));
@@ -83,7 +84,7 @@ public class TestServiceManager {
initUpgrade(context, "v2", false, true, false);
ServiceManager manager = context.getServiceManager();
//make components stable by upgrading all instances
- upgradeAllInstances(context);
+ upgradeAndReadyAllInstances(context);
GenericTestUtils.waitFor(()->
context.service.getState().equals(ServiceState.STABLE),
@@ -115,7 +116,7 @@ public class TestServiceManager {
manager.getServiceSpec().getState());
//make components stable by upgrading all instances
- upgradeAllInstances(context);
+ upgradeAndReadyAllInstances(context);
// finalize service
context.scheduler.getDispatcher().getEventHandler().handle(
@@ -138,7 +139,7 @@ public class TestServiceManager {
initUpgrade(context, "v2", true, true, false);
// make components stable
- upgradeAllInstances(context);
+ upgradeAndReadyAllInstances(context);
GenericTestUtils.waitFor(() ->
context.service.getState().equals(ServiceState.STABLE),
@@ -174,18 +175,17 @@ public class TestServiceManager {
public void testExpressUpgrade() throws Exception {
ServiceContext context = createServiceContext("testExpressUpgrade");
ServiceManager manager = context.getServiceManager();
- manager.getServiceSpec().setState(
- ServiceState.EXPRESS_UPGRADING);
+ manager.getServiceSpec().setState(ServiceState.EXPRESS_UPGRADING);
initUpgrade(context, "v2", true, true, true);
List<String> comps = ServiceApiUtil.resolveCompsDependency(context.service);
- // wait till instances of first component are in upgrade
- String comp1 = comps.get(0);
- upgradeInstancesOf(context, comp1);
+ // wait till instances of first component are upgraded and ready
+ String compA = comps.get(0);
+ makeInstancesReadyAfterUpgrade(context, compA);
- // wait till instances of second component are in upgrade
- String comp2 = comps.get(1);
- upgradeInstancesOf(context, comp2);
+ // wait till instances of second component are upgraded and ready
+ String compB = comps.get(1);
+ makeInstancesReadyAfterUpgrade(context, compB);
GenericTestUtils.waitFor(() ->
context.service.getState().equals(ServiceState.STABLE),
@@ -196,6 +196,57 @@ public class TestServiceManager {
validateUpgradeFinalization(manager.getName(), "v2");
}
+ @Test(timeout = TIMEOUT)
+ public void testCancelUpgrade() throws Exception {
+ ServiceContext context = createServiceContext("testCancelUpgrade");
+ writeInitialDef(context.service);
+ initUpgrade(context, "v2", true, false, false);
+ ServiceManager manager = context.getServiceManager();
+ Assert.assertEquals("service not upgrading", ServiceState.UPGRADING,
+ manager.getServiceSpec().getState());
+
+ List<String> comps = ServiceApiUtil.resolveCompsDependency(context.service);
+ // wait till instances of first component are upgraded and ready
+ String compA = comps.get(0);
+ // upgrade the instances
+ upgradeInstances(context, compA);
+ makeInstancesReadyAfterUpgrade(context, compA);
+
+ // cancel upgrade
+ context.scheduler.getDispatcher().getEventHandler().handle(
+ new ServiceEvent(ServiceEventType.CANCEL_UPGRADE));
+ makeInstancesReadyAfterUpgrade(context, compA);
+
+ GenericTestUtils.waitFor(()->
+ context.service.getState().equals(ServiceState.STABLE),
+ CHECK_EVERY_MILLIS, TIMEOUT);
+ Assert.assertEquals("service upgrade not cancelled", ServiceState.STABLE,
+ manager.getServiceSpec().getState());
+
+ validateUpgradeFinalization(manager.getName(), "v1");
+ }
+
+ @Test(timeout = TIMEOUT)
+ public void testCancelUpgradeAfterInitiate() throws Exception {
+ ServiceContext context = createServiceContext("testCancelUpgrade");
+ writeInitialDef(context.service);
+ initUpgrade(context, "v2", true, false, false);
+ ServiceManager manager = context.getServiceManager();
+ Assert.assertEquals("service not upgrading", ServiceState.UPGRADING,
+ manager.getServiceSpec().getState());
+
+ // cancel upgrade
+ context.scheduler.getDispatcher().getEventHandler().handle(
+ new ServiceEvent(ServiceEventType.CANCEL_UPGRADE));
+ GenericTestUtils.waitFor(()->
+ context.service.getState().equals(ServiceState.STABLE),
+ CHECK_EVERY_MILLIS, TIMEOUT);
+ Assert.assertEquals("service upgrade not cancelled", ServiceState.STABLE,
+ manager.getServiceSpec().getState());
+
+ validateUpgradeFinalization(manager.getName(), "v1");
+ }
+
private void validateUpgradeFinalization(String serviceName,
String expectedVersion) throws IOException {
Service savedSpec = ServiceApiUtil.loadService(rule.getFs(), serviceName);
@@ -225,21 +276,23 @@ public class TestServiceManager {
}
writeUpgradedDef(upgradedDef);
serviceManager.processUpgradeRequest(version, autoFinalize, expressUpgrade);
- ServiceEvent upgradeEvent = new ServiceEvent(ServiceEventType.UPGRADE);
- upgradeEvent.setVersion(version).setExpressUpgrade(expressUpgrade)
- .setAutoFinalize(autoFinalize);
-
- GenericTestUtils.waitFor(()-> {
- ServiceState serviceState = context.service.getState();
- if (serviceState.equals(ServiceState.UPGRADING) ||
- serviceState.equals(ServiceState.UPGRADING_AUTO_FINALIZE) ||
- serviceState.equals(ServiceState.EXPRESS_UPGRADING)) {
- return true;
+ GenericTestUtils.waitFor(() -> {
+ for (Component comp : context.scheduler.getAllComponents().values()) {
+ if (!comp.getComponentSpec().getState().equals(
+ ComponentState.NEEDS_UPGRADE)) {
+ return false;
+ }
}
- return false;
+ return true;
}, CHECK_EVERY_MILLIS, TIMEOUT);
}
+ private void upgradeAndReadyAllInstances(ServiceContext context) throws
+ TimeoutException, InterruptedException {
+ upgradeAllInstances(context);
+ makeAllInstancesReady(context);
+ }
+
private void upgradeAllInstances(ServiceContext context) throws
TimeoutException, InterruptedException {
// upgrade the instances
@@ -248,8 +301,10 @@ public class TestServiceManager {
ComponentInstanceEventType.UPGRADE);
context.scheduler.getDispatcher().getEventHandler().handle(event);
}));
+ }
- // become ready
+ private void makeAllInstancesReady(ServiceContext context)
+ throws TimeoutException, InterruptedException {
context.scheduler.getLiveInstances().forEach(((containerId, instance) -> {
ComponentInstanceEvent event = new ComponentInstanceEvent(containerId,
ComponentInstanceEventType.BECOME_READY);
@@ -267,7 +322,19 @@ public class TestServiceManager {
}, CHECK_EVERY_MILLIS, TIMEOUT);
}
- private void upgradeInstancesOf(ServiceContext context, String compName)
+ private void upgradeInstances(ServiceContext context, String compName) {
+ Collection<ComponentInstance> compInstances = context.scheduler
+ .getAllComponents().get(compName).getAllComponentInstances();
+ compInstances.forEach(instance -> {
+ ComponentInstanceEvent event = new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.UPGRADE);
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
+ });
+ }
+
+ private void makeInstancesReadyAfterUpgrade(ServiceContext context,
+ String compName)
throws TimeoutException, InterruptedException {
Collection<ComponentInstance> compInstances = context.scheduler
.getAllComponents().get(compName).getAllComponentInstances();
@@ -289,6 +356,15 @@ public class TestServiceManager {
context.scheduler.getDispatcher().getEventHandler().handle(event);
});
+
+ GenericTestUtils.waitFor(() -> {
+ for (ComponentInstance instance : compInstances) {
+ if (!instance.getContainerState().equals(ContainerState.READY)) {
+ return false;
+ }
+ }
+ return true;
+ }, CHECK_EVERY_MILLIS, TIMEOUT);
}
private ServiceContext createServiceContext(String name)
@@ -324,6 +400,14 @@ public class TestServiceManager {
return artifact;
}
+ private void writeInitialDef(Service service)
+ throws IOException, SliderException {
+ Path servicePath = rule.getFs().buildClusterDirPath(
+ service.getName());
+ ServiceApiUtil.createDirAndPersistApp(rule.getFs(), servicePath,
+ service);
+ }
+
private void writeUpgradedDef(Service upgradedDef)
throws IOException, SliderException {
Path upgradePath = rule.getFs().buildClusterUpgradeDirPath(
@@ -332,6 +416,6 @@ public class TestServiceManager {
upgradedDef);
}
- private static final int TIMEOUT = 200000;
+ private static final int TIMEOUT = 10000;
private static final int CHECK_EVERY_MILLIS = 100;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 216d88f..3e23a10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service;
+import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
@@ -450,6 +451,49 @@ public class TestYarnNativeServices extends ServiceTestUtils {
client.actionDestroy(service.getName());
}
+ @Test(timeout = 200000)
+ public void testCancelUpgrade() throws Exception {
+ setupInternal(NUM_NMS);
+ getConf().setBoolean(YARN_SERVICE_UPGRADE_ENABLED, true);
+ ServiceClient client = createClient(getConf());
+
+ Service service = createExampleApplication();
+ Component component = service.getComponents().iterator().next();
+ component.getConfiguration().getEnv().put("key1", "val0");
+
+ client.actionCreate(service);
+ waitForServiceToBeStable(client, service);
+
+ // upgrade the service
+ service.setState(ServiceState.UPGRADING);
+ service.setVersion("v2");
+ component.getConfiguration().getEnv().put("key1", "val1");
+ client.initiateUpgrade(service);
+
+ // wait for service to be in upgrade state
+ waitForServiceToBeInState(client, service, ServiceState.UPGRADING);
+
+ // upgrade 1 container
+ Service liveService = client.getStatus(service.getName());
+ Container container = liveService.getComponent(component.getName())
+ .getContainers().iterator().next();
+ client.actionUpgrade(service, Lists.newArrayList(container));
+
+ Thread.sleep(500);
+ // cancel the upgrade
+ client.actionCancelUpgrade(service.getName());
+ waitForServiceToBeStable(client, service);
+ Service active = client.getStatus(service.getName());
+ Assert.assertEquals("component not stable", ComponentState.STABLE,
+ active.getComponent(component.getName()).getState());
+ Assert.assertEquals("comp does not have new env", "val0",
+ active.getComponent(component.getName()).getConfiguration()
+ .getEnv("key1"));
+ LOG.info("Stop/destroy service {}", service);
+ client.actionStop(service.getName(), true);
+ client.actionDestroy(service.getName());
+ }
+
// Test to verify ANTI_AFFINITY placement policy
// 1. Start mini cluster with 3 NMs and scheduler placement-constraint handler
// 2. Create an example service with 3 containers
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index 0e047c2..41be8c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -221,6 +221,17 @@ public class TestServiceCLI {
Assert.assertEquals(result, 0);
}
+ @Test
+ public void testCancelUpgrade() throws Exception {
+ conf.set(YARN_APP_ADMIN_CLIENT_PREFIX + DUMMY_APP_TYPE,
+ DummyServiceClient.class.getName());
+ cli.setConf(conf);
+ String[] args = {"app", "-upgrade", "app-1",
+ "-cancel", "-appTypes", DUMMY_APP_TYPE};
+ int result = cli.run(ApplicationCLI.preProcessArgs(args));
+ Assert.assertEquals(result, 0);
+ }
+
@Test (timeout = 180000)
public void testEnableFastLaunch() throws Exception {
fs.getFileSystem().create(new Path(basedir.getAbsolutePath(), "test.jar"))
@@ -332,5 +343,11 @@ public class TestServiceCLI {
throws IOException, YarnException {
return "";
}
+
+ @Override
+ public int actionCancelUpgrade(String appName) throws IOException,
+ YarnException {
+ return 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
index e1a4d9d..f11d871 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.yarn.service.component;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.service.MockRunningServiceContext;
import org.apache.hadoop.yarn.service.ServiceContext;
import org.apache.hadoop.yarn.service.ServiceTestUtils;
import org.apache.hadoop.yarn.service.TestServiceManager;
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
-import org.apache.hadoop.yarn.service.MockRunningServiceContext;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Rule;
@@ -38,9 +37,8 @@ import org.junit.Test;
import java.util.Iterator;
+import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.BECOME_READY;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
/**
* Tests for {@link Component}.
@@ -78,14 +76,14 @@ public class TestComponent {
"val1")).setUpgradeVersion("v2"));
// one instance finished upgrading
- comp.decContainersThatNeedUpgrade();
+ comp.getUpgradeStatus().decContainersThatNeedUpgrade();
comp.handle(new ComponentEvent(comp.getName(),
ComponentEventType.CHECK_STABLE));
Assert.assertEquals("component not in need upgrade state",
ComponentState.NEEDS_UPGRADE, comp.getComponentSpec().getState());
// second instance finished upgrading
- comp.decContainersThatNeedUpgrade();
+ comp.getUpgradeStatus().decContainersThatNeedUpgrade();
comp.handle(new ComponentEvent(comp.getName(),
ComponentEventType.CHECK_STABLE));
@@ -97,7 +95,7 @@ public class TestComponent {
@Test
public void testContainerCompletedWhenUpgrading() throws Exception {
- String serviceName = "testContainerComplete";
+ String serviceName = "testContainerCompletedWhenUpgrading";
MockRunningServiceContext context = createTestContext(rule, serviceName);
Component comp = context.scheduler.getAllComponents().entrySet().iterator()
.next().getValue();
@@ -105,48 +103,233 @@ public class TestComponent {
comp.handle(new ComponentEvent(comp.getName(), ComponentEventType.UPGRADE)
.setTargetSpec(createSpecWithEnv(serviceName, comp.getName(), "key1",
"val1")).setUpgradeVersion("v2"));
- comp.getAllComponentInstances().forEach(instance -> {
- instance.handle(new ComponentInstanceEvent(
- instance.getContainer().getId(), ComponentInstanceEventType.UPGRADE));
- });
- Iterator<ComponentInstance> instanceIter = comp.
- getAllComponentInstances().iterator();
+ comp.getAllComponentInstances().forEach(instance ->
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(), ComponentInstanceEventType.UPGRADE)));
// reinitialization of a container failed
- ContainerStatus status = mock(ContainerStatus.class);
- when(status.getExitStatus()).thenReturn(ContainerExitStatus.ABORTED);
- ComponentInstance instance = instanceIter.next();
+ for(ComponentInstance instance : comp.getAllComponentInstances()) {
+ ComponentEvent stopEvent = new ComponentEvent(comp.getName(),
+ ComponentEventType.CONTAINER_COMPLETED)
+ .setInstance(instance)
+ .setContainerId(instance.getContainer().getId());
+ comp.handle(stopEvent);
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(), STOP));
+ }
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CHECK_STABLE));
+
+ Assert.assertEquals("component not in needs upgrade state",
+ ComponentState.NEEDS_UPGRADE, comp.getComponentSpec().getState());
+ }
+
+ @Test
+ public void testCancelUpgrade() throws Exception {
+ ServiceContext context = createTestContext(rule, "testCancelUpgrade");
+ Component comp = context.scheduler.getAllComponents().entrySet().iterator()
+ .next().getValue();
+
+ ComponentEvent upgradeEvent = new ComponentEvent(comp.getName(),
+ ComponentEventType.CANCEL_UPGRADE);
+ comp.handle(upgradeEvent);
+ Assert.assertEquals("component not in need upgrade state",
+ ComponentState.NEEDS_UPGRADE, comp.getComponentSpec().getState());
+
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.service.component.ComponentState
+ .CANCEL_UPGRADING, comp.getState());
+ }
+
+ @Test
+ public void testContainerCompletedCancelUpgrade() throws Exception {
+ String serviceName = "testContainerCompletedCancelUpgrade";
+ MockRunningServiceContext context = createTestContext(rule, serviceName);
+ Component comp = context.scheduler.getAllComponents().entrySet().iterator()
+ .next().getValue();
+
+ // upgrade completes
+ comp.handle(new ComponentEvent(comp.getName(), ComponentEventType.UPGRADE)
+ .setTargetSpec(createSpecWithEnv(serviceName, comp.getName(), "key1",
+ "val1")).setUpgradeVersion("v2"));
+ comp.getAllComponentInstances().forEach(instance ->
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.UPGRADE)));
+
+ // reinitialization of a container done
+ for(ComponentInstance instance : comp.getAllComponentInstances()) {
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(), BECOME_READY));
+ }
+
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CANCEL_UPGRADE)
+ .setTargetSpec(createSpecWithEnv(serviceName, comp.getName(), "key1",
+ "val0")).setUpgradeVersion("v1"));
+ comp.getAllComponentInstances().forEach(instance ->
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.CANCEL_UPGRADE)));
+
+ Iterator<ComponentInstance> iter = comp.getAllComponentInstances()
+ .iterator();
+
+ // cancel upgrade failed of a container
+ ComponentInstance instance1 = iter.next();
ComponentEvent stopEvent = new ComponentEvent(comp.getName(),
ComponentEventType.CONTAINER_COMPLETED)
- .setInstance(instance).setContainerId(instance.getContainer().getId())
- .setStatus(status);
+ .setInstance(instance1)
+ .setContainerId(instance1.getContainer().getId());
comp.handle(stopEvent);
- instance.handle(new ComponentInstanceEvent(instance.getContainer().getId(),
- STOP).setStatus(status));
+ instance1.handle(new ComponentInstanceEvent(
+ instance1.getContainer().getId(), STOP));
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.service.component.ComponentState
+ .CANCEL_UPGRADING, comp.getState());
comp.handle(new ComponentEvent(comp.getName(),
ComponentEventType.CHECK_STABLE));
- Assert.assertEquals("component not in flexing state",
- ComponentState.FLEXING, comp.getComponentSpec().getState());
-
- // new container get allocated
- context.assignNewContainer(context.attemptId, 10, comp);
+ Assert.assertEquals("component not in needs upgrade state",
+ ComponentState.NEEDS_UPGRADE, comp.getComponentSpec().getState());
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.service.component.ComponentState
+ .CANCEL_UPGRADING, comp.getState());
// second instance finished upgrading
- ComponentInstance instance2 = instanceIter.next();
+ ComponentInstance instance2 = iter.next();
instance2.handle(new ComponentInstanceEvent(
instance2.getContainer().getId(),
ComponentInstanceEventType.BECOME_READY));
+
comp.handle(new ComponentEvent(comp.getName(),
ComponentEventType.CHECK_STABLE));
+ Assert.assertEquals("component not in flexing state",
+ ComponentState.FLEXING, comp.getComponentSpec().getState());
+ // new container get allocated
+ context.assignNewContainer(context.attemptId, 10, comp);
+
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CHECK_STABLE));
+
Assert.assertEquals("component not in stable state",
ComponentState.STABLE, comp.getComponentSpec().getState());
- Assert.assertEquals("component did not upgrade successfully", "val1",
+ Assert.assertEquals("cancel upgrade failed", "val0",
+ comp.getComponentSpec().getConfiguration().getEnv("key1"));
+ }
+
+ @Test
+ public void testCancelUpgradeSuccessWhileUpgrading() throws Exception {
+ String serviceName = "testCancelUpgradeWhileUpgrading";
+ MockRunningServiceContext context = createTestContext(rule, serviceName);
+ Component comp = context.scheduler.getAllComponents().entrySet().iterator()
+ .next().getValue();
+ cancelUpgradeWhileUpgrading(context, comp);
+
+ // cancel upgrade successful for both instances
+ for(ComponentInstance instance : comp.getAllComponentInstances()) {
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.BECOME_READY));
+ }
+
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CHECK_STABLE));
+
+ Assert.assertEquals("component not in stable state",
+ ComponentState.STABLE, comp.getComponentSpec().getState());
+ Assert.assertEquals("cancel upgrade failed", "val0",
+ comp.getComponentSpec().getConfiguration().getEnv("key1"));
+ }
+
+ @Test
+ public void testCancelUpgradeFailureWhileUpgrading() throws Exception {
+ String serviceName = "testCancelUpgradeFailureWhileUpgrading";
+ MockRunningServiceContext context = createTestContext(rule, serviceName);
+ Component comp = context.scheduler.getAllComponents().entrySet().iterator()
+ .next().getValue();
+ cancelUpgradeWhileUpgrading(context, comp);
+
+ // cancel upgrade failed for both instances
+ for(ComponentInstance instance : comp.getAllComponentInstances()) {
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.STOP));
+ }
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CHECK_STABLE));
+
+ Assert.assertEquals("component not in flexing state",
+ ComponentState.FLEXING, comp.getComponentSpec().getState());
+
+ for (ComponentInstance instance : comp.getAllComponentInstances()) {
+ // new container get allocated
+ context.assignNewContainer(context.attemptId, 10, comp);
+ }
+
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CHECK_STABLE));
+
+ Assert.assertEquals("component not in stable state",
+ ComponentState.STABLE, comp.getComponentSpec().getState());
+ Assert.assertEquals("cancel upgrade failed", "val0",
comp.getComponentSpec().getConfiguration().getEnv("key1"));
}
+ private void cancelUpgradeWhileUpgrading(
+ MockRunningServiceContext context, Component comp)
+ throws Exception {
+
+ comp.handle(new ComponentEvent(comp.getName(), ComponentEventType.UPGRADE)
+ .setTargetSpec(createSpecWithEnv(context.service.getName(),
+ comp.getName(), "key1", "val1")).setUpgradeVersion("v0"));
+
+ Iterator<ComponentInstance> iter = comp.getAllComponentInstances()
+ .iterator();
+
+ ComponentInstance instance1 = iter.next();
+
+ // instance1 is triggered to upgrade
+ instance1.handle(new ComponentInstanceEvent(
+ instance1.getContainer().getId(), ComponentInstanceEventType.UPGRADE));
+
+ // component upgrade is cancelled
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CANCEL_UPGRADE)
+ .setTargetSpec(createSpecWithEnv(context.service.getName(),
+ comp.getName(), "key1",
+ "val0")).setUpgradeVersion("v0"));
+
+ // all instances upgrade is cancelled.
+ comp.getAllComponentInstances().forEach(instance ->
+ instance.handle(new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.CANCEL_UPGRADE)));
+
+ // regular upgrade failed for instance 1
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CONTAINER_COMPLETED).setInstance(instance1)
+ .setContainerId(instance1.getContainer().getId()));
+ instance1.handle(new ComponentInstanceEvent(
+ instance1.getContainer().getId(), STOP));
+
+ // component should be in cancel upgrade
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.service.component.ComponentState
+ .CANCEL_UPGRADING, comp.getState());
+
+ comp.handle(new ComponentEvent(comp.getName(),
+ ComponentEventType.CHECK_STABLE));
+
+ Assert.assertEquals("component not in needs upgrade state",
+ ComponentState.NEEDS_UPGRADE, comp.getComponentSpec().getState());
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.service.component.ComponentState
+ .CANCEL_UPGRADING, comp.getState());
+ }
+
@Test
public void testComponentStateReachesStableStateWithTerminatingComponents()
throws
@@ -249,8 +432,6 @@ public class TestComponent {
serviceState);
}
-
-
private static org.apache.hadoop.yarn.service.api.records.Component
createSpecWithEnv(String serviceName, String compName, String key,
String val) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
index e039981..c5a9631 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/instance/TestComponentInstance.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.service.component.instance;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -86,7 +87,7 @@ public class TestComponentInstance {
@Test
public void testContainerReadyAfterUpgrade() throws Exception {
ServiceContext context = TestComponent.createTestContext(rule,
- "testContainerStarted");
+ "testContainerReadyAfterUpgrade");
Component component = context.scheduler.getAllComponents().entrySet()
.iterator().next().getValue();
upgradeComponent(component);
@@ -105,12 +106,186 @@ public class TestComponentInstance {
.getId().toString()).getState());
}
+
+ @Test
+ public void testContainerUpgradeFailed() throws Exception {
+ ServiceContext context = TestComponent.createTestContext(rule,
+ "testContainerUpgradeFailed");
+ Component component = context.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ upgradeComponent(component);
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+
+ ComponentInstanceEvent upgradeEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(), ComponentInstanceEventType.UPGRADE);
+ instance.handle(upgradeEvent);
+
+ ContainerStatus containerStatus = mock(ContainerStatus.class);
+ when(containerStatus.getExitStatus()).thenReturn(
+ ContainerExitStatus.ABORTED);
+ ComponentInstanceEvent stopEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(), ComponentInstanceEventType.STOP)
+ .setStatus(containerStatus);
+ // this is the call back from NM for the upgrade
+ instance.handle(stopEvent);
+ Assert.assertEquals("instance did not fail", ContainerState.FAILED_UPGRADE,
+ component.getComponentSpec().getContainer(instance.getContainer()
+ .getId().toString()).getState());
+ }
+
+ @Test
+ public void testCancelNothingToUpgrade() throws Exception {
+ ServiceContext context = TestComponent.createTestContext(rule,
+ "testCancelUpgradeWhenContainerReady");
+ Component component = context.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ cancelCompUpgrade(component);
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+
+ ComponentInstanceEvent cancelEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.CANCEL_UPGRADE);
+ instance.handle(cancelEvent);
+
+ Assert.assertEquals("instance not ready", ContainerState.READY,
+ component.getComponentSpec().getContainer(instance.getContainer()
+ .getId().toString()).getState());
+ }
+
+ @Test
+ public void testCancelUpgradeFailed() throws Exception {
+ ServiceContext context = TestComponent.createTestContext(rule,
+ "testCancelUpgradeFailed");
+ Component component = context.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ cancelCompUpgrade(component);
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+
+ ComponentInstanceEvent cancelEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.CANCEL_UPGRADE);
+ instance.handle(cancelEvent);
+
+ instance.handle(new ComponentInstanceEvent(instance.getContainer().getId(),
+ ComponentInstanceEventType.STOP));
+ Assert.assertEquals("instance not init", ComponentInstanceState.INIT,
+ instance.getState());
+ }
+
+ @Test
+ public void testCancelAfterCompProcessedCancel() throws Exception {
+ ServiceContext context = TestComponent.createTestContext(rule,
+ "testCancelAfterCompProcessedCancel");
+ Component component = context.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ upgradeComponent(component);
+ cancelCompUpgrade(component);
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+ ComponentInstanceEvent upgradeEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(), ComponentInstanceEventType.UPGRADE);
+ instance.handle(upgradeEvent);
+
+ Assert.assertEquals("instance should start upgrading",
+ ContainerState.NEEDS_UPGRADE,
+ component.getComponentSpec().getContainer(instance.getContainer()
+ .getId().toString()).getState());
+ }
+
+ @Test
+ public void testCancelWhileUpgradeWithSuccess() throws Exception {
+ validateCancelWhileUpgrading(true, true);
+ }
+
+ @Test
+ public void testCancelWhileUpgradeWithFailure() throws Exception {
+ validateCancelWhileUpgrading(false, true);
+ }
+
+ @Test
+ public void testCancelFailedWhileUpgradeWithSuccess() throws Exception {
+ validateCancelWhileUpgrading(true, false);
+ }
+
+ @Test
+ public void testCancelFailedWhileUpgradeWithFailure() throws Exception {
+ validateCancelWhileUpgrading(false, false);
+ }
+
+ private void validateCancelWhileUpgrading(boolean upgradeSuccessful,
+ boolean cancelUpgradeSuccessful)
+ throws Exception {
+ ServiceContext context = TestComponent.createTestContext(rule,
+ "testCancelWhileUpgrading");
+ Component component = context.scheduler.getAllComponents().entrySet()
+ .iterator().next().getValue();
+ upgradeComponent(component);
+
+ ComponentInstance instance = component.getAllComponentInstances().iterator()
+ .next();
+ ComponentInstanceEvent upgradeEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(), ComponentInstanceEventType.UPGRADE);
+ instance.handle(upgradeEvent);
+
+ Assert.assertEquals("instance should be upgrading",
+ ContainerState.UPGRADING,
+ component.getComponentSpec().getContainer(instance.getContainer()
+ .getId().toString()).getState());
+
+ cancelCompUpgrade(component);
+ ComponentInstanceEvent cancelEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ ComponentInstanceEventType.CANCEL_UPGRADE);
+ instance.handle(cancelEvent);
+
+ // either upgrade failed or successful
+ ComponentInstanceEvent readyOrStopEvent = new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ upgradeSuccessful ? ComponentInstanceEventType.BECOME_READY :
+ ComponentInstanceEventType.STOP);
+
+ instance.handle(readyOrStopEvent);
+ Assert.assertEquals("instance not upgrading", ContainerState.UPGRADING,
+ component.getComponentSpec().getContainer(instance.getContainer()
+ .getId().toString()).getState());
+
+ // response for cancel received
+ ComponentInstanceEvent readyOrStopCancel = new ComponentInstanceEvent(
+ instance.getContainer().getId(),
+ cancelUpgradeSuccessful ? ComponentInstanceEventType.BECOME_READY :
+ ComponentInstanceEventType.STOP);
+
+ instance.handle(readyOrStopCancel);
+ if (cancelUpgradeSuccessful) {
+ Assert.assertEquals("instance not ready", ContainerState.READY,
+ component.getComponentSpec().getContainer(instance.getContainer()
+ .getId().toString()).getState());
+ } else {
+ Assert.assertEquals("instance not init", ComponentInstanceState.INIT,
+ instance.getState());
+ }
+ }
+
private void upgradeComponent(Component component) {
component.handle(new ComponentEvent(component.getName(),
ComponentEventType.UPGRADE).setTargetSpec(component.getComponentSpec())
.setUpgradeVersion("v2"));
}
+ private void cancelCompUpgrade(Component component) {
+ component.handle(new ComponentEvent(component.getName(),
+ ComponentEventType.CANCEL_UPGRADE)
+ .setTargetSpec(component.getComponentSpec())
+ .setUpgradeVersion("v1"));
+ }
+
private Component createComponent(ServiceScheduler scheduler,
org.apache.hadoop.yarn.service.api.records.Component.RestartPolicyEnum
restartPolicy, int nSucceededInstances, int nFailedInstances,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/log4j.properties
new file mode 100644
index 0000000..81a3f6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index a0e4e02..b0e12bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -102,6 +102,7 @@ public class ApplicationCLI extends YarnCLI {
public static final String ENABLE_FAST_LAUNCH = "enableFastLaunch";
public static final String UPGRADE_CMD = "upgrade";
public static final String UPGRADE_EXPRESS = "express";
+ public static final String UPGRADE_CANCEL = "cancel";
public static final String UPGRADE_INITIATE = "initiate";
public static final String UPGRADE_AUTO_FINALIZE = "autoFinalize";
public static final String UPGRADE_FINALIZE = "finalize";
@@ -265,6 +266,8 @@ public class ApplicationCLI extends YarnCLI {
opts.addOption(UPGRADE_AUTO_FINALIZE, false, "Works with -upgrade and " +
"-initiate options to initiate the upgrade of the application with " +
"the ability to finalize the upgrade automatically.");
+ opts.addOption(UPGRADE_CANCEL, false, "Works with -upgrade option to " +
+ "cancel current upgrade.");
opts.getOption(LAUNCH_CMD).setArgName("Application Name> <File Name");
opts.getOption(LAUNCH_CMD).setArgs(2);
opts.getOption(START_CMD).setArgName("Application Name");
@@ -646,7 +649,7 @@ public class ApplicationCLI extends YarnCLI {
} else if (cliParser.hasOption(UPGRADE_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, UPGRADE_CMD, UPGRADE_EXPRESS,
UPGRADE_INITIATE, UPGRADE_AUTO_FINALIZE, UPGRADE_FINALIZE,
- COMPONENT_INSTS, COMPONENTS, APP_TYPE_CMD)) {
+ UPGRADE_CANCEL, COMPONENT_INSTS, COMPONENTS, APP_TYPE_CMD)) {
printUsage(title, opts);
return exitCode;
}
@@ -697,6 +700,13 @@ public class ApplicationCLI extends YarnCLI {
return exitCode;
}
return client.actionStart(appName);
+ } else if (cliParser.hasOption(UPGRADE_CANCEL)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, UPGRADE_CMD,
+ UPGRADE_CANCEL, APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ return client.actionCancelUpgrade(appName);
}
} else {
syserr.println("Invalid Command Usage : ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index a600895..f795db5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -2143,6 +2143,8 @@ public class TestYarnCLI {
pw.println(" the upgrade of the application");
pw.println(" with the ability to finalize the");
pw.println(" upgrade automatically.");
+ pw.println(" -cancel Works with -upgrade option to");
+ pw.println(" cancel current upgrade.");
pw.println(" -changeQueue <Queue Name> Moves application to a new");
pw.println(" queue. ApplicationId can be");
pw.println(" passed using 'appId' option.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
index 232666d..df11ffd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -300,4 +300,17 @@ public abstract class AppAdminClient extends CompositeService {
@Unstable
public abstract int actionUpgradeExpress(String appName, File fileName)
throws IOException, YarnException;
+
+ /**
+ * Cancels the upgrade of the service.
+ *
+ * @param appName the name of the application
+ * @return exit code
+ * @throws IOException
+ * @throws YarnException
+ */
+ @Public
+ @Unstable
+ public abstract int actionCancelUpgrade(String appName) throws IOException,
+ YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 27a7c80..01d70af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1836,6 +1836,7 @@ public class ContainerManagerImpl extends CompositeService implements
public void reInitializeContainer(ContainerId containerId,
ContainerLaunchContext reInitLaunchContext, boolean autoCommit)
throws YarnException {
+ LOG.debug("{} requested reinit", containerId);
Container container = preReInitializeOrLocalizeCheck(containerId,
ReInitOp.RE_INIT);
ResourceSet resourceSet = new ResourceSet();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-8734. Readiness check for
remote service belonging to the same user. Contributed by Eric Yang
Posted by in...@apache.org.
YARN-8734. Readiness check for remote service belonging to the same user. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1c1dde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1c1dde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1c1dde3
Branch: refs/heads/HDFS-13532
Commit: d1c1dde3097eea6a46a39d0312f94949f6884002
Parents: e211350
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Sep 28 09:39:59 2018 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Fri Sep 28 09:40:48 2018 -0700
----------------------------------------------------------------------
...RN-Simplified-V1-API-Layer-For-Services.yaml | 5 +++
.../hadoop/yarn/service/ServiceManager.java | 3 +-
.../hadoop/yarn/service/ServiceScheduler.java | 2 +
.../yarn/service/api/records/Service.java | 15 +++++++
.../yarn/service/utils/ServiceApiUtil.java | 43 ++++++++++++++++++++
.../yarn/service/TestYarnNativeServices.java | 2 +
.../yarn/service/utils/TestServiceApiUtil.java | 39 ++++++++++++++++++
.../markdown/yarn-service/YarnServiceAPI.md | 2 +-
8 files changed, 109 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index d90ae06..d74604a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -254,6 +254,11 @@ definitions:
docker_client_config:
type: string
description: URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json).
+ dependencies:
+ type: array
+ items:
+ type: string
+ description: An array of services which should be in STABLE state, before this service can be started.
ResourceInformation:
description:
ResourceInformation determines unit/value of resource types in addition to memory and vcores. It will be part of Resource object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
index 4851325..aefdadd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
@@ -170,7 +170,8 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
} else {
serviceManager.setServiceState(ServiceState.UPGRADING);
}
-
+ ServiceApiUtil.checkServiceDependencySatisified(serviceManager
+ .getServiceSpec());
return State.UPGRADING;
} catch (Throwable e) {
LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index b49ef2a..9b9305c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -389,6 +389,8 @@ public class ServiceScheduler extends CompositeService {
// Since AM has been started and registered, the service is in STARTED state
app.setState(ServiceState.STARTED);
+ ServiceApiUtil.checkServiceDependencySatisified(context.service);
+
// recover components based on containers sent from RM
recoverComponents(response);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
index 57d1a1a..6e03121 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -73,6 +73,7 @@ public class Service extends BaseResource {
private String version = null;
private String description = null;
private String dockerClientConfig = null;
+ private List<String> dependencies = new ArrayList<String>();
/**
* A unique service name.
@@ -352,6 +353,18 @@ public class Service extends BaseResource {
this.queue = queue;
}
+ @ApiModelProperty(example = "null", value = "A list of dependent services.")
+ @XmlElement(name = "dependencies")
+ @JsonProperty("dependencies")
+ public List<String> getDependencies() {
+ return dependencies;
+ }
+
+ public void setDependencies(List<String>
+ dependencies) {
+ this.dependencies = dependencies;
+ }
+
public Service kerberosPrincipal(KerberosPrincipal kerberosPrincipal) {
this.kerberosPrincipal = kerberosPrincipal;
return this;
@@ -437,6 +450,8 @@ public class Service extends BaseResource {
.append(toIndentedString(kerberosPrincipal)).append("\n");
sb.append(" dockerClientConfig: ")
.append(toIndentedString(dockerClientConfig)).append("\n");
+ sb.append(" dependencies: ")
+ .append(toIndentedString(dependencies)).append("\n");
sb.append("}");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 0eb54ce..d201c7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.service.api.records.ComponentState;
import org.apache.hadoop.yarn.service.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Configuration;
@@ -705,4 +707,45 @@ public class ServiceApiUtil {
}
return components;
}
+
+ private static boolean serviceDependencySatisfied(Service service) {
+ boolean result = true;
+ try {
+ List<String> dependencies = service
+ .getDependencies();
+ org.apache.hadoop.conf.Configuration conf =
+ new org.apache.hadoop.conf.Configuration();
+ if (dependencies != null && dependencies.size() > 0) {
+ ServiceClient sc = new ServiceClient();
+ sc.init(conf);
+ sc.start();
+ for (String dependent : dependencies) {
+ Service dependentService = sc.getStatus(dependent);
+ if (dependentService.getState() == null ||
+ !dependentService.getState().equals(ServiceState.STABLE)) {
+ result = false;
+ LOG.info("Service dependency is not satisfied for " +
+ "service: {} state: {}", dependent,
+ dependentService.getState());
+ }
+ }
+ sc.close();
+ }
+ } catch (IOException | YarnException e) {
+ LOG.warn("Caught exception: ", e);
+ LOG.info("Service dependency is not satisified.");
+ result = false;
+ }
+ return result;
+ }
+
+ public static void checkServiceDependencySatisified(Service service) {
+ while (!serviceDependencySatisfied(service)) {
+ try {
+ LOG.info("Waiting for service dependencies.");
+ Thread.sleep(15000L);
+ } catch (InterruptedException e) {
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 3e23a10..d50c574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -438,6 +438,8 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// wait for upgrade to complete
waitForServiceToBeStable(client, service);
Service active = client.getStatus(service.getName());
+ Assert.assertEquals("version mismatch", service.getVersion(),
+ active.getVersion());
Assert.assertEquals("component not stable", ComponentState.STABLE,
active.getComponent(component.getName()).getState());
Assert.assertEquals("compa does not have new env", "val1",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
index 98e7474..1e3c180 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.PlacementScope;
import org.apache.hadoop.yarn.service.api.records.PlacementType;
import org.apache.hadoop.yarn.service.api.records.Resource;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -733,6 +734,44 @@ public class TestServiceApiUtil extends ServiceTestUtils {
}
}
+ @Test(timeout = 1500)
+ public void testNoServiceDependencies() {
+ Service service = createExampleApplication();
+ Component compa = createComponent("compa");
+ Component compb = createComponent("compb");
+ service.addComponent(compa);
+ service.addComponent(compb);
+ List<String> dependencies = new ArrayList<String>();
+ service.setDependencies(dependencies);
+ ServiceApiUtil.checkServiceDependencySatisified(service);
+ }
+
+ @Test
+ public void testServiceDependencies() {
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ Service service = createExampleApplication();
+ Component compa = createComponent("compa");
+ Component compb = createComponent("compb");
+ service.addComponent(compa);
+ service.addComponent(compb);
+ List<String> dependencies = new ArrayList<String>();
+ dependencies.add("abc");
+ service.setDependencies(dependencies);
+ Service dependent = createExampleApplication();
+ dependent.setState(ServiceState.STOPPED);
+ ServiceApiUtil.checkServiceDependencySatisified(service);
+ }
+ };
+ thread.start();
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ Assert.assertTrue(thread.isAlive());
+ }
+
public static Service createExampleApplication() {
Service exampleApp = new Service();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index 4bfa742..7b1e74a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -406,7 +406,7 @@ a service resource has the following attributes.
|queue|The YARN queue that this service should be submitted to.|false|string||
|kerberos_principal|The principal info of the user who launches the service|false|KerberosPrincipal||
|docker_client_config|URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json)|false|string||
-
+|dependencies|A list of service names that this service depends on.| false | string array ||
### ServiceState
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-15693. Typo in Secure Mode
documentation. Contributed by Thomas Levine.
Posted by in...@apache.org.
HADOOP-15693. Typo in Secure Mode documentation. Contributed by Thomas Levine.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c62b4404
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c62b4404
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c62b4404
Branch: refs/heads/HDFS-13532
Commit: c62b4404219df8fb0edbaf71e80862c0dcce2b40
Parents: 44edcdf
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Sep 26 10:30:43 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Sep 26 10:30:43 2018 +0900
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/SecureMode.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c62b4404/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index 3dd4af6..ecf94eb 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -37,7 +37,7 @@ When service level authentication is turned on, end users must authenticate them
Ensure that HDFS and YARN daemons run as different Unix users, e.g. `hdfs` and `yarn`. Also, ensure that the MapReduce JobHistory server runs as different user such as `mapred`.
-It's recommended to have them share a Unix group, for e.g. `hadoop`. See also "[Mapping from user to group](#Mapping_from_user_to_group)" for group management.
+It's recommended to have them share a Unix group, e.g. `hadoop`. See also "[Mapping from user to group](#Mapping_from_user_to_group)" for group management.
| User:Group | Daemons |
|:--------------|:----------------------------------------------------|
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-13943. [JDK10] Fix javadoc
errors in hadoop-hdfs-client module. Contributed by Akira Ajisaka.
Posted by in...@apache.org.
HDFS-13943. [JDK10] Fix javadoc errors in hadoop-hdfs-client module. Contributed by Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6c5ef99
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6c5ef99
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6c5ef99
Branch: refs/heads/HDFS-13532
Commit: f6c5ef9903dba5eb268997110ef169125327c2c8
Parents: 7d08219
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Tue Oct 2 09:49:48 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Tue Oct 2 09:49:48 2018 +0900
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/XAttr.java | 2 +-
.../org/apache/hadoop/hdfs/AddBlockFlag.java | 8 ++-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 6 +-
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 2 +-
.../java/org/apache/hadoop/hdfs/DFSPacket.java | 16 +++---
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 4 +-
.../hadoop/hdfs/DistributedFileSystem.java | 13 ++---
.../hadoop/hdfs/NameNodeProxiesClient.java | 3 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 12 ++--
.../hadoop/hdfs/client/impl/LeaseRenewer.java | 2 +-
.../hdfs/protocol/ClientDatanodeProtocol.java | 2 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 30 +++++-----
.../hdfs/protocol/ReconfigurationProtocol.java | 2 +-
.../token/block/BlockTokenIdentifier.java | 4 +-
.../hdfs/shortcircuit/DfsClientShmManager.java | 5 +-
.../hadoop/hdfs/util/ByteArrayManager.java | 4 +-
.../hadoop/hdfs/util/StripedBlockUtil.java | 59 +++++++++++---------
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 1 -
18 files changed, 92 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
index de9bbda..a06ff0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
* not visible to the user except when getXAttr/getXAttrs is called on a file
* or directory in the /.reserved/raw HDFS directory hierarchy. These
* attributes can only be accessed by the superuser.
- * <p/>
+ * <p>
* @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
* http://en.wikipedia.org/wiki/Extended_file_attributes</a>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
index b0686d7..1b6ee4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/AddBlockFlag.java
@@ -20,11 +20,17 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+
+import java.util.EnumSet;
/**
* AddBlockFlag provides hints for new block allocation and placement.
* Users can use this flag to control <em>per DFSOutputStream</em>
- * {@see ClientProtocol#addBlock()} behavior.
+ * @see ClientProtocol#addBlock(String, String, ExtendedBlock, DatanodeInfo[],
+ * long, String[], EnumSet)
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 38072b2..5511657 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1054,8 +1054,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
/**
* Call {@link #create(String, boolean, short, long, Progressable)} with
- * default <code>replication</code> and <code>blockSize<code> and null <code>
- * progress</code>.
+ * default <code>replication</code> and <code>blockSize</code> and null
+ * <code>progress</code>.
*/
public OutputStream create(String src, boolean overwrite)
throws IOException {
@@ -1065,7 +1065,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
/**
* Call {@link #create(String, boolean, short, long, Progressable)} with
- * default <code>replication</code> and <code>blockSize<code>.
+ * default <code>replication</code> and <code>blockSize</code>.
*/
public OutputStream create(String src,
boolean overwrite, Progressable progress) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
index bcdc1c8..a921a19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
@@ -139,7 +139,7 @@ public class DFSInotifyEventInputStream {
* are falling behind (i.e. transaction are being generated faster than the
* client is reading them). If a client falls too far behind events may be
* deleted before the client can read them.
- * <p/>
+ * <p>
* A return value of -1 indicates that an estimate could not be produced, and
* should be ignored. The value returned by this method is really only useful
* when compared to previous or subsequent returned values.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
index 7fe1278..272d8de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
@@ -294,12 +294,12 @@ public class DFSPacket {
}
/**
- * Add a trace parent span for this packet.<p/>
- *
+ * Add a trace parent span for this packet.
+ * <p>
* Trace parent spans for a packet are the trace spans responsible for
* adding data to that packet. We store them as an array of longs for
- * efficiency.<p/>
- *
+ * efficiency.
+ * <p>
* Protected by the DFSOutputStream dataQueue lock.
*/
public void addTraceParent(Span span) {
@@ -323,10 +323,10 @@ public class DFSPacket {
}
/**
- * Get the trace parent spans for this packet.<p/>
- *
- * Will always be non-null.<p/>
- *
+ * Get the trace parent spans for this packet.
+ * <p>
+ * Will always be non-null.
+ * <p>
* Protected by the DFSOutputStream dataQueue lock.
*/
public SpanId[] getTraceParents() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index 8f785c7..0686ed5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -266,8 +266,8 @@ public class DFSUtilClient {
}
/**
- * Convert a List<LocatedBlock> to BlockLocation[]
- * @param blocks A List<LocatedBlock> to be converted
+ * Convert a List to BlockLocation[]
+ * @param blocks A List to be converted
* @return converted array of BlockLocation
*/
public static BlockLocation[] locatedBlocks2Locations(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 28c1e27..3532a71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -149,7 +149,6 @@ public class DistributedFileSystem extends FileSystem
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
*
* @return <code>hdfs</code>
*/
@@ -1860,7 +1859,7 @@ public class DistributedFileSystem extends FileSystem
return setSafeMode(SafeModeAction.SAFEMODE_GET, true);
}
- /** @see HdfsAdmin#allowSnapshot(Path) */
+ /** @see org.apache.hadoop.hdfs.client.HdfsAdmin#allowSnapshot(Path) */
public void allowSnapshot(final Path path) throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.ALLOW_SNAPSHOT);
@@ -1888,7 +1887,7 @@ public class DistributedFileSystem extends FileSystem
}.resolve(this, absF);
}
- /** @see HdfsAdmin#disallowSnapshot(Path) */
+ /** @see org.apache.hadoop.hdfs.client.HdfsAdmin#disallowSnapshot(Path) */
public void disallowSnapshot(final Path path) throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.DISALLOW_SNAPSHOT);
@@ -2207,7 +2206,7 @@ public class DistributedFileSystem extends FileSystem
}
/**
- * @see {@link #addCacheDirective(CacheDirectiveInfo, EnumSet)}
+ * @see #addCacheDirective(CacheDirectiveInfo, EnumSet)
*/
public long addCacheDirective(CacheDirectiveInfo info) throws IOException {
return addCacheDirective(info, EnumSet.noneOf(CacheFlag.class));
@@ -2234,7 +2233,7 @@ public class DistributedFileSystem extends FileSystem
}
/**
- * @see {@link #modifyCacheDirective(CacheDirectiveInfo, EnumSet)}
+ * @see #modifyCacheDirective(CacheDirectiveInfo, EnumSet)
*/
public void modifyCacheDirective(CacheDirectiveInfo info) throws IOException {
modifyCacheDirective(info, EnumSet.noneOf(CacheFlag.class));
@@ -3305,10 +3304,10 @@ public class DistributedFileSystem extends FileSystem
* Returns a RemoteIterator which can be used to list all open files
* currently managed by the NameNode. For large numbers of open files,
* iterator will fetch the list in batches of configured size.
- * <p/>
+ * <p>
* Since the list is fetched in batches, it does not represent a
* consistent snapshot of the all open files.
- * <p/>
+ * <p>
* This method can only be called by HDFS superusers.
*/
@Deprecated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
index a092f02..897ecc8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
@@ -67,7 +67,7 @@ import org.apache.hadoop.security.UserGroupInformation;
* non-HA-enabled client proxy as appropriate.
*
* For creating proxy objects with other protocols, please see
- * {@link NameNodeProxies#createProxy(Configuration, URI, Class)}.
+ * NameNodeProxies#createProxy(Configuration, URI, Class).
*/
@InterfaceAudience.Private
public class NameNodeProxiesClient {
@@ -118,7 +118,6 @@ public class NameNodeProxiesClient {
* @return an object containing both the proxy and the associated
* delegation token service it corresponds to
* @throws IOException if there is an error creating the proxy
- * @see {@link NameNodeProxies#createProxy(Configuration, URI, Class)}.
*/
public static ProxyAndInfo<ClientProtocol> createProxyWithClientProtocol(
Configuration conf, URI nameNodeUri, AtomicBoolean fallbackToSimpleAuth)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index f204f3c..575cd24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -360,10 +360,10 @@ public class HdfsAdmin {
* Returns a RemoteIterator which can be used to list the encryption zones
* in HDFS. For large numbers of encryption zones, the iterator will fetch
* the list of zones in a number of small batches.
- * <p/>
+ * <p>
* Since the list is fetched in batches, it does not represent a
* consistent snapshot of the entire list of encryption zones.
- * <p/>
+ * <p>
* This method can only be called by HDFS superusers.
*/
public RemoteIterator<EncryptionZone> listEncryptionZones()
@@ -418,7 +418,7 @@ public class HdfsAdmin {
* for information on stream usage.
* See {@link org.apache.hadoop.hdfs.inotify.Event}
* for information on the available events.
- * <p/>
+ * <p>
* Inotify users may want to tune the following HDFS parameters to
* ensure that enough extra HDFS edits are saved to support inotify clients
* that fall behind the current state of the namespace while reading events.
@@ -438,7 +438,7 @@ public class HdfsAdmin {
* dfs.namenode.checkpoint.txns
* dfs.namenode.num.checkpoints.retained
* dfs.ha.log-roll.period
- * <p/>
+ * <p>
* It is recommended that local journaling be configured
* (dfs.namenode.edits.dir) for inotify (in addition to a shared journal)
* so that edit transfers from the shared journal can be avoided.
@@ -615,10 +615,10 @@ public class HdfsAdmin {
* Returns a RemoteIterator which can be used to list all open files
* currently managed by the NameNode. For large numbers of open files,
* iterator will fetch the list in batches of configured size.
- * <p/>
+ * <p>
* Since the list is fetched in batches, it does not represent a
* consistent snapshot of the all open files.
- * <p/>
+ * <p>
* This method can only be called by HDFS superusers.
*/
@Deprecated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
index 957c0a9..e8ec41c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
@@ -70,7 +70,7 @@ import org.slf4j.LoggerFactory;
* to renew the leases.
* </li>
* </ul>
- * </p>
+ * <p>
*/
@InterfaceAudience.Private
public class LeaseRenewer {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
index e2be9c6..7312f26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
@@ -143,7 +143,7 @@ public interface ClientDatanodeProtocol {
/**
* Get the status of the previously issued reconfig task.
- * @see {@link org.apache.hadoop.conf.ReconfigurationTaskStatus}.
+ * @see org.apache.hadoop.conf.ReconfigurationTaskStatus
*/
ReconfigurationTaskStatus getReconfigurationStatus() throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index e8c881b..a55a0f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -839,7 +839,7 @@ public interface ClientProtocol {
* percentage called threshold of blocks, which satisfy the minimal
* replication condition.
* The minimal replication condition is that each block must have at least
- * <tt>dfs.namenode.replication.min</tt> replicas.
+ * {@code dfs.namenode.replication.min} replicas.
* When the threshold is reached the name node extends safe mode
* for a configurable amount of time
* to let the remaining data nodes to check in before it
@@ -855,12 +855,13 @@ public interface ClientProtocol {
* Current state of the name node can be verified using
* {@link #setSafeMode(HdfsConstants.SafeModeAction,boolean)
* setSafeMode(SafeModeAction.SAFEMODE_GET,false)}
- * <h4>Configuration parameters:</h4>
- * <tt>dfs.safemode.threshold.pct</tt> is the threshold parameter.<br>
- * <tt>dfs.safemode.extension</tt> is the safe mode extension parameter.<br>
- * <tt>dfs.namenode.replication.min</tt> is the minimal replication parameter.
*
- * <h4>Special cases:</h4>
+ * <p><b>Configuration parameters:</b></p>
+ * {@code dfs.safemode.threshold.pct} is the threshold parameter.<br>
+ * {@code dfs.safemode.extension} is the safe mode extension parameter.<br>
+ * {@code dfs.namenode.replication.min} is the minimal replication parameter.
+ *
+ * <p><b>Special cases:</b></p>
* The name node does not enter safe mode at startup if the threshold is
* set to 0 or if the name space is empty.<br>
* If the threshold is set to 1 then all blocks need to have at least
@@ -1211,7 +1212,6 @@ public interface ClientProtocol {
* Get a valid Delegation Token.
*
* @param renewer the designated renewer for the token
- * @return Token<DelegationTokenIdentifier>
* @throws IOException
*/
@Idempotent
@@ -1490,7 +1490,7 @@ public interface ClientProtocol {
throws IOException;
/**
- * Used to implement cursor-based batched listing of {@EncryptionZone}s.
+ * Used to implement cursor-based batched listing of {@link EncryptionZone}s.
*
* @param prevId ID of the last item in the previous batch. If there is no
* previous batch, a negative value can be used.
@@ -1513,7 +1513,7 @@ public interface ClientProtocol {
/**
* Used to implement cursor-based batched listing of
- * {@ZoneReencryptionStatus}s.
+ * {@link ZoneReencryptionStatus}s.
*
* @param prevId ID of the last item in the previous batch. If there is no
* previous batch, a negative value can be used.
@@ -1528,7 +1528,7 @@ public interface ClientProtocol {
* Set xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param src file or directory
@@ -1545,12 +1545,12 @@ public interface ClientProtocol {
* If xAttrs is null or empty, this is the same as getting all xattrs of the
* file or directory. Only those xattrs for which the logged-in user has
* permissions to view are returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param src file or directory
* @param xAttrs xAttrs to get
- * @return List<XAttr> <code>XAttr</code> list
+ * @return <code>XAttr</code> list
* @throws IOException
*/
@Idempotent
@@ -1561,11 +1561,11 @@ public interface ClientProtocol {
* List the xattrs names for a file or directory.
* Only the xattr names for which the logged in user has the permissions to
* access will be returned.
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param src file or directory
- * @return List<XAttr> <code>XAttr</code> list
+ * @return <code>XAttr</code> list
* @throws IOException
*/
@Idempotent
@@ -1576,7 +1576,7 @@ public interface ClientProtocol {
* Remove xattr of a file or directory.Value in xAttr parameter is ignored.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
- * <p/>
+ * <p>
* Refer to the HDFS extended attributes user documentation for details.
*
* @param src file or directory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReconfigurationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReconfigurationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReconfigurationProtocol.java
index 8370438..540a472 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReconfigurationProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReconfigurationProtocol.java
@@ -45,7 +45,7 @@ public interface ReconfigurationProtocol {
/**
* Get the status of the previously issued reconfig task.
- * @see {@link org.apache.hadoop.conf.ReconfigurationTaskStatus}.
+ * @see org.apache.hadoop.conf.ReconfigurationTaskStatus
*/
@Idempotent
ReconfigurationTaskStatus getReconfigurationStatus() throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
index 5950752..0635fd0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
@@ -182,13 +182,13 @@ public class BlockTokenIdentifier extends TokenIdentifier {
* because we know the first field is the Expiry date.
*
* In the case of the legacy buffer, the expiry date is a VInt, so the size
- * (which should always be >1) is encoded in the first byte - which is
+ * (which should always be >1) is encoded in the first byte - which is
* always negative due to this encoding. However, there are sometimes null
* BlockTokenIdentifier written so we also need to handle the case there
* the first byte is also 0.
*
* In the case of protobuf, the first byte is a type tag for the expiry date
- * which is written as <code>(field_number << 3 | wire_type</code>.
+ * which is written as <code>field_number << 3 | wire_type</code>.
* So as long as the field_number is less than 16, but also positive, then
* we know we have a Protobuf.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
index 2262003..6b3d8e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
@@ -57,9 +57,8 @@ import org.slf4j.LoggerFactory;
* slots in the set of existing segments, falling back to getting a new segment
* from the DataNode via {@link DataTransferProtocol#requestShortCircuitFds}.
*
- * The counterpart to this class on the DataNode is
- * {@link ShortCircuitRegistry}. See {@link ShortCircuitRegistry} for more
- * information on the communication protocol.
+ * The counterpart to this class on the DataNode is ShortCircuitRegistry.
+ * See ShortCircuitRegistry for more information on the communication protocol.
*/
@InterfaceAudience.Private
public class DfsClientShmManager implements Closeable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
index a9adb7e..164b03b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
@@ -54,10 +54,10 @@ public abstract class ByteArrayManager {
/**
* @return the least power of two greater than or equal to n, i.e. return
- * the least integer x with x >= n and x a power of two.
+ * the least integer x with x >= n and x a power of two.
*
* @throws HadoopIllegalArgumentException
- * if n <= 0.
+ * if n <= 0.
*/
public static int leastPowerOfTwo(final int n) {
if (n <= 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 9bad45d..0425927 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -53,7 +53,8 @@ import java.util.concurrent.TimeUnit;
* illustrated in the following diagram. Unless otherwise specified, all
* range-related calculations are inclusive (the end offset of the previous
* range should be 1 byte lower than the start offset of the next one).
- *
+ */
+ /*
* | <---- Block Group ----> | <- Block Group: logical unit composing
* | | striped HDFS files.
* blk_0 blk_1 blk_2 <- Internal Blocks: each internal block
@@ -492,9 +493,12 @@ public class StripedBlockUtil {
return stripes.toArray(new AlignedStripe[stripes.size()]);
}
+ /**
+ * Cell indexing convention defined in {@link StripingCell}.
+ */
private static void calcualteChunkPositionsInBuf(int cellSize,
AlignedStripe[] stripes, StripingCell[] cells, ByteBuffer buf) {
- /**
+ /*
* | <--------------- AlignedStripe --------------->|
*
* |<- length_0 ->|<-- length_1 -->|<- length_2 ->|
@@ -508,8 +512,6 @@ public class StripedBlockUtil {
* | cell_0_0_0 | cell_1_0_1 and cell_2_0_2 |cell_3_1_0 ...| <- buf
* | (partial) | (from blk_1 and blk_2) | |
* +----------------------------------------------------------+
- *
- * Cell indexing convention defined in {@link StripingCell}
*/
int done = 0;
for (StripingCell cell : cells) {
@@ -562,7 +564,11 @@ public class StripedBlockUtil {
* its start and end offsets -- e.g., the end logical offset of cell_0_0_0
* should be 1 byte lower than the start logical offset of cell_1_0_1.
*
- * | <------- Striped Block Group -------> |
+ * A StripingCell is a special instance of {@link StripingChunk} whose offset
+ * and size align with the cell used when writing data.
+ * TODO: consider parity cells
+ */
+ /* | <------- Striped Block Group -------> |
* blk_0 blk_1 blk_2
* | | |
* v v v
@@ -572,9 +578,6 @@ public class StripedBlockUtil {
* |cell_3_1_0| |cell_4_1_1| |cell_5_1_2| <- {@link #idxInBlkGroup} = 5
* +----------+ +----------+ +----------+ {@link #idxInInternalBlk} = 1
* {@link #idxInStripe} = 2
- * A StripingCell is a special instance of {@link StripingChunk} whose offset
- * and size align with the cell used when writing data.
- * TODO: consider parity cells
*/
@VisibleForTesting
public static class StripingCell {
@@ -622,6 +625,18 @@ public class StripedBlockUtil {
* the diagram, any given byte range on a block group leads to 1~5
* AlignedStripe's.
*
+ * An AlignedStripe is the basic unit of reading from a striped block group,
+ * because within the AlignedStripe, all internal blocks can be processed in
+ * a uniform manner.
+ *
+ * The coverage of an AlignedStripe on an internal block is represented as a
+ * {@link StripingChunk}.
+ *
+ * To simplify the logic of reading a logical byte range from a block group,
+ * a StripingChunk is either completely in the requested byte range or
+ * completely outside the requested byte range.
+ */
+ /*
* |<-------- Striped Block Group -------->|
* blk_0 blk_1 blk_2 blk_3 blk_4
* +----+ | +----+ +----+
@@ -638,18 +653,7 @@ public class StripedBlockUtil {
* | | | | | | | <- AlignedStripe4:
* +----+ | +----+ +----+ last cell is partial
* |
- * <---- data blocks ----> | <--- parity --->
- *
- * An AlignedStripe is the basic unit of reading from a striped block group,
- * because within the AlignedStripe, all internal blocks can be processed in
- * a uniform manner.
- *
- * The coverage of an AlignedStripe on an internal block is represented as a
- * {@link StripingChunk}.
- *
- * To simplify the logic of reading a logical byte range from a block group,
- * a StripingChunk is either completely in the requested byte range or
- * completely outside the requested byte range.
+ * <---- data blocks ----> | <--- parity -->
*/
public static class AlignedStripe {
public VerticalRange range;
@@ -691,7 +695,8 @@ public class StripedBlockUtil {
* starting at {@link #offsetInBlock} and lasting for {@link #spanInBlock}
* bytes in an internal block. Note that VerticalRange doesn't necessarily
* align with {@link StripingCell}.
- *
+ */
+ /*
* |<- Striped Block Group ->|
* blk_0
* |
@@ -735,8 +740,8 @@ public class StripedBlockUtil {
/**
* Indicates the coverage of an {@link AlignedStripe} on an internal block,
* and the state of the chunk in the context of the read request.
- *
- * |<---------------- Striped Block Group --------------->|
+ */
+ /* |<---------------- Striped Block Group --------------->|
* blk_0 blk_1 blk_2 blk_3 blk_4
* +---------+ | +----+ +----+
* null null |REQUESTED| | |null| |null| <- AlignedStripe0
@@ -745,7 +750,7 @@ public class StripedBlockUtil {
* +---------+ +---------+ +---------+ | +----+ +----+
* |REQUESTED| |REQUESTED| ALLZERO | |null| |null| <- AlignedStripe2
* +---------+ +---------+ | +----+ +----+
- * <----------- data blocks ------------> | <--- parity --->
+ * <----------- data blocks ------------> | <--- parity -->
*/
public static class StripingChunk {
/** Chunk has been successfully fetched */
@@ -767,10 +772,12 @@ public class StripedBlockUtil {
/**
* If a chunk is completely in requested range, the state transition is:
- * REQUESTED (when AlignedStripe created) -> PENDING -> {FETCHED | MISSING}
+ * REQUESTED (when AlignedStripe created) -> PENDING ->
+ * {FETCHED | MISSING}
* If a chunk is completely outside requested range (including parity
* chunks), state transition is:
- * null (AlignedStripe created) -> REQUESTED (upon failure) -> PENDING ...
+ * null (AlignedStripe created) ->REQUESTED (upon failure) ->
+ * PENDING ...
*/
public int state = REQUESTED;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c5ef99/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index ec60a18..d504cfe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -176,7 +176,6 @@ public class WebHdfsFileSystem extends FileSystem
/**
* Return the protocol scheme for the FileSystem.
- * <p/>
*
* @return <code>webhdfs</code>
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HADOOP-15801. ABFS: Fixing
skipUserGroupMetadata in AzureBlobFileSystemStore. Contributed by Da Zhou
Posted by in...@apache.org.
HADOOP-15801. ABFS: Fixing skipUserGroupMetadata in AzureBlobFileSystemStore.
Contributed by Da Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a383ac47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a383ac47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a383ac47
Branch: refs/heads/HDFS-13532
Commit: a383ac47cabe22e563725fae9de73e5e69e51f3e
Parents: 43bc984
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 2 11:42:52 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 2 11:42:52 2018 +0100
----------------------------------------------------------------------
.../fs/azurebfs/AzureBlobFileSystemStore.java | 27 ++++++++++++++------
1 file changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a383ac47/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index cf7387b..a735ce0 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -102,6 +102,8 @@ public class AzureBlobFileSystemStore {
private AbfsClient client;
private URI uri;
private final UserGroupInformation userGroupInformation;
+ private final String userName;
+ private final String primaryUserGroup;
private static final String DATE_TIME_PATTERN = "E, dd MMM yyyy HH:mm:ss 'GMT'";
private static final String XMS_PROPERTIES_ENCODING = "ISO-8859-1";
private static final int LIST_MAX_RESULTS = 5000;
@@ -128,6 +130,15 @@ public class AzureBlobFileSystemStore {
}
this.userGroupInformation = userGroupInformation;
+ this.userName = userGroupInformation.getShortUserName();
+
+ if (!abfsConfiguration.getSkipUserGroupMetadataDuringInitialization()) {
+ primaryUserGroup = userGroupInformation.getPrimaryGroupName();
+ } else {
+ //Provide a default group name
+ primaryUserGroup = userName;
+ }
+
this.azureAtomicRenameDirSet = new HashSet<>(Arrays.asList(
abfsConfiguration.getAzureAtomicRenameDirs().split(AbfsHttpConstants.COMMA)));
@@ -462,8 +473,8 @@ public class AzureBlobFileSystemStore {
final boolean hasAcl = AbfsPermission.isExtendedAcl(permissions);
return new VersionedFileStatus(
- owner == null ? userGroupInformation.getUserName() : owner,
- group == null ? userGroupInformation.getPrimaryGroupName() : group,
+ owner == null ? userName : owner,
+ group == null ? primaryUserGroup : group,
permissions == null ? new AbfsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)
: AbfsPermission.valueOf(permissions),
hasAcl,
@@ -489,8 +500,8 @@ public class AzureBlobFileSystemStore {
final boolean hasAcl = AbfsPermission.isExtendedAcl(permissions);
return new VersionedFileStatus(
- owner == null ? userGroupInformation.getUserName() : owner,
- group == null ? userGroupInformation.getPrimaryGroupName() : group,
+ owner == null ? userName : owner,
+ group == null ? primaryUserGroup : group,
permissions == null ? new AbfsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)
: AbfsPermission.valueOf(permissions),
hasAcl,
@@ -528,8 +539,8 @@ public class AzureBlobFileSystemStore {
long blockSize = abfsConfiguration.getAzureBlockSize();
for (ListResultEntrySchema entry : retrievedSchema.paths()) {
- final String owner = entry.owner() == null ? userGroupInformation.getUserName() : entry.owner();
- final String group = entry.group() == null ? userGroupInformation.getPrimaryGroupName() : entry.group();
+ final String owner = entry.owner() == null ? userName : entry.owner();
+ final String group = entry.group() == null ? primaryUserGroup : entry.group();
final FsPermission fsPermission = entry.permissions() == null
? new AbfsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)
: AbfsPermission.valueOf(entry.permissions());
@@ -761,8 +772,8 @@ public class AzureBlobFileSystemStore {
: AbfsPermission.valueOf(permissions);
final AclStatus.Builder aclStatusBuilder = new AclStatus.Builder();
- aclStatusBuilder.owner(owner == null ? userGroupInformation.getUserName() : owner);
- aclStatusBuilder.group(group == null ? userGroupInformation.getPrimaryGroupName() : group);
+ aclStatusBuilder.owner(owner == null ? userName : owner);
+ aclStatusBuilder.group(group == null ? primaryUserGroup : group);
aclStatusBuilder.setPermission(fsPermission);
aclStatusBuilder.stickyBit(fsPermission.getStickyBit());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDDS-448. Move NodeStat to
NodeStatemanager from SCMNodeManager. Contributed by LiXin Ge.
Posted by in...@apache.org.
HDDS-448. Move NodeStat to NodeStatemanager from SCMNodeManager. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c798c1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c798c1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c798c1e
Branch: refs/heads/HDFS-13532
Commit: 3c798c1e3c75a7d689a42ab01a649f93643239fa
Parents: d1c1dde
Author: Ajay Kumar <aj...@apache.com>
Authored: Fri Sep 28 10:33:17 2018 -0700
Committer: Ajay Kumar <aj...@apache.com>
Committed: Fri Sep 28 10:33:17 2018 -0700
----------------------------------------------------------------------
.../placement/algorithms/SCMCommonPolicy.java | 4 +-
.../placement/metrics/SCMNodeMetric.java | 1 +
.../hadoop/hdds/scm/node/NodeManager.java | 3 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 54 ++++++++++++++--
.../hadoop/hdds/scm/node/SCMNodeManager.java | 49 ++++++++------
.../hdds/scm/node/states/NodeStateMap.java | 68 ++++++++++++++++++--
.../hdds/scm/container/MockNodeManager.java | 9 ++-
.../hdds/scm/node/TestNodeReportHandler.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 3 +-
9 files changed, 155 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
index ba241dc..60861b7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
@@ -150,8 +150,8 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
private boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
long sizeRequired) {
SCMNodeMetric nodeMetric = nodeManager.getNodeStat(datanodeDetails);
- return (nodeMetric != null) && nodeMetric.get().getRemaining()
- .hasResources(sizeRequired);
+ return (nodeMetric != null) && (nodeMetric.get() != null)
+ && nodeMetric.get().getRemaining().hasResources(sizeRequired);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java
index b50376d..efd5fd6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java
@@ -55,6 +55,7 @@ public class SCMNodeMetric implements DatanodeMetric<SCMNodeStat, Long> {
*/
@Override
public boolean isGreater(SCMNodeStat o) {
+ Preconditions.checkNotNull(this.stat, "Argument cannot be null");
Preconditions.checkNotNull(o, "Argument cannot be null");
// if zero, replace with 1 for the division to work.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index fc73253..fad3ee3 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -121,7 +121,8 @@ public interface NodeManager extends StorageContainerNodeProtocol,
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails DatanodeDetails.
- * @return node stat if it is live/stale, null if it is dead or does't exist.
+ * @return node stat if it is live/stale, null if it is decommissioned or
+ * doesn't exist.
*/
SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index 77f939e..1f99ffe 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.HddsServerUtil;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
@@ -39,13 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
@@ -417,6 +412,51 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
+ * Returns the current stats of the node.
+ *
+ * @param uuid node id
+ *
+ * @return SCMNodeStat
+ *
+ * @throws NodeNotFoundException if the node is not present
+ */
+ public SCMNodeStat getNodeStat(UUID uuid) throws NodeNotFoundException {
+ return nodeStateMap.getNodeStat(uuid);
+ }
+
+ /**
+ * Returns a unmodifiable copy of nodeStats.
+ * @return map with node stats.
+ */
+ public Map<UUID, SCMNodeStat> getNodeStatsMap() {
+ return nodeStateMap.getNodeStats();
+ }
+
+ /**
+ * Set the stat for the node.
+ *
+ * @param uuid node id.
+ *
+ * @param newstat new stat that will set to the specify node.
+ */
+ public void setNodeStat(UUID uuid, SCMNodeStat newstat) {
+ nodeStateMap.setNodeStat(uuid, newstat);
+ }
+
+ /**
+ * Remove the current stats of the specify node.
+ *
+ * @param uuid node id
+ *
+ * @return SCMNodeStat the stat removed from the node.
+ *
+ * @throws NodeNotFoundException if the node is not present.
+ */
+ public SCMNodeStat removeNodeStat(UUID uuid) throws NodeNotFoundException {
+ return nodeStateMap.removeNodeStat(uuid);
+ }
+
+ /**
* Move Stale or Dead node to healthy if we got a heartbeat from them.
* Move healthy nodes to stale nodes if it is needed.
* Move Stales node to dead if needed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index fd5f5de..620c816 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -56,12 +56,10 @@ import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
import java.io.IOException;
import java.net.InetAddress;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -85,11 +83,7 @@ public class SCMNodeManager
static final Logger LOG =
LoggerFactory.getLogger(SCMNodeManager.class);
-
private final NodeStateManager nodeStateManager;
- // Individual live node stats
- // TODO: NodeStat should be moved to NodeStatemanager (NodeStateMap)
- private final ConcurrentHashMap<UUID, SCMNodeStat> nodeStats;
// Should we maintain aggregated stats? If this is not frequently used, we
// can always calculate it from nodeStats whenever required.
// Aggregated node stats
@@ -124,7 +118,6 @@ public class SCMNodeManager
StorageContainerManager scmManager, EventPublisher eventPublisher)
throws IOException {
this.nodeStateManager = new NodeStateManager(conf, eventPublisher);
- this.nodeStats = new ConcurrentHashMap<>();
this.scmStat = new SCMNodeStat();
this.clusterID = clusterID;
this.version = VersionInfo.getLatestVersion();
@@ -297,8 +290,10 @@ public class SCMNodeManager
private void updateNodeStat(UUID dnId, NodeReportProto nodeReport) {
- SCMNodeStat stat = nodeStats.get(dnId);
- if (stat == null) {
+ SCMNodeStat stat;
+ try {
+ stat = nodeStateManager.getNodeStat(dnId);
+ } catch (NodeNotFoundException e) {
LOG.debug("SCM updateNodeStat based on heartbeat from previous" +
"dead datanode {}", dnId);
stat = new SCMNodeStat();
@@ -317,9 +312,9 @@ public class SCMNodeManager
}
scmStat.subtract(stat);
stat.set(totalCapacity, totalScmUsed, totalRemaining);
- nodeStats.put(dnId, stat);
scmStat.add(stat);
}
+ nodeStateManager.setNodeStat(dnId, stat);
}
/**
@@ -377,7 +372,7 @@ public class SCMNodeManager
UUID dnId = datanodeDetails.getUuid();
try {
nodeStateManager.addNode(datanodeDetails);
- nodeStats.put(dnId, new SCMNodeStat());
+ nodeStateManager.setNodeStat(dnId, new SCMNodeStat());
if(inStartupChillMode.get() &&
nodeStateManager.getTotalNodeCount() >= getMinimumChillModeNodes()) {
inStartupChillMode.getAndSet(false);
@@ -446,17 +441,25 @@ public class SCMNodeManager
*/
@Override
public Map<UUID, SCMNodeStat> getNodeStats() {
- return Collections.unmodifiableMap(nodeStats);
+ return nodeStateManager.getNodeStatsMap();
}
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails - datanode ID.
- * @return node stat if it is live/stale, null if it is dead or does't exist.
+ * @return node stat if it is live/stale, null if it is decommissioned or
+ * doesn't exist.
*/
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) {
- return new SCMNodeMetric(nodeStats.get(datanodeDetails.getUuid()));
+ try {
+ return new SCMNodeMetric(
+ nodeStateManager.getNodeStat(datanodeDetails.getUuid()));
+ } catch (NodeNotFoundException e) {
+ LOG.info("SCM getNodeStat from a decommissioned or removed datanode {}",
+ datanodeDetails.getUuid());
+ return null;
+ }
}
@Override
@@ -492,18 +495,22 @@ public class SCMNodeManager
}
/**
- * Remove the node stats and update the storage stats
- * in this SCM Node Manager.
+ * Update the node stats and cluster storage stats in this SCM Node Manager.
*
* @param dnUuid datanode uuid.
*/
@Override
public void processDeadNode(UUID dnUuid) {
- SCMNodeStat stat = nodeStats.get(dnUuid);
- LOG.trace("Update stat values as Datanode {} is dead.", dnUuid);
- if (stat != null) {
- scmStat.subtract(stat);
- stat.set(0, 0, 0);
+ try {
+ SCMNodeStat stat = nodeStateManager.getNodeStat(dnUuid);
+ if (stat != null) {
+ LOG.trace("Update stat values as Datanode {} is dead.", dnUuid);
+ scmStat.subtract(stat);
+ stat.set(0, 0, 0);
+ }
+ } catch (NodeNotFoundException e) {
+ LOG.warn("Can't update stats based on message of dead Datanode {}, it"
+ + " doesn't exist or decommissioned already.", dnUuid);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
index dd91866..774ced1 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/NodeStateMap.java
@@ -20,14 +20,10 @@ package org.apache.hadoop.hdds.scm.node.states;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -48,6 +44,11 @@ public class NodeStateMap {
* Represents the current state of node.
*/
private final ConcurrentHashMap<NodeState, Set<UUID>> stateMap;
+ /**
+ * Represents the current stats of node.
+ */
+ private final ConcurrentHashMap<UUID, SCMNodeStat> nodeStats;
+
private final ReadWriteLock lock;
/**
@@ -57,6 +58,7 @@ public class NodeStateMap {
lock = new ReentrantReadWriteLock();
nodeMap = new ConcurrentHashMap<>();
stateMap = new ConcurrentHashMap<>();
+ nodeStats = new ConcurrentHashMap<>();
initStateMap();
}
@@ -260,6 +262,60 @@ public class NodeStateMap {
}
/**
+ * Returns the current stats of the node.
+ *
+ * @param uuid node id
+ *
+ * @return SCMNodeStat of the specify node.
+ *
+ * @throws NodeNotFoundException if the node is not found
+ */
+ public SCMNodeStat getNodeStat(UUID uuid) throws NodeNotFoundException {
+ SCMNodeStat stat = nodeStats.get(uuid);
+ if (stat == null) {
+ throw new NodeNotFoundException("Node UUID: " + uuid);
+ }
+ return stat;
+ }
+
+ /**
+ * Returns a unmodifiable copy of nodeStats.
+ *
+ * @return map with node stats.
+ */
+ public Map<UUID, SCMNodeStat> getNodeStats() {
+ return Collections.unmodifiableMap(nodeStats);
+ }
+
+ /**
+ * Set the current stats of the node.
+ *
+ * @param uuid node id
+ *
+ * @param newstat stat that will set to the specify node.
+ */
+ public void setNodeStat(UUID uuid, SCMNodeStat newstat) {
+ nodeStats.put(uuid, newstat);
+ }
+
+ /**
+ * Remove the current stats of the specify node.
+ *
+ * @param uuid node id
+ *
+ * @return SCMNodeStat the stat removed from the node.
+ *
+ * @throws NodeNotFoundException if the node is not found
+ */
+ public SCMNodeStat removeNodeStat(UUID uuid) throws NodeNotFoundException {
+ SCMNodeStat stat = nodeStats.remove(uuid);
+ if (stat == null) {
+ throw new NodeNotFoundException("Node UUID: " + uuid);
+ }
+ return stat;
+ }
+
+ /**
* Since we don't hold a global lock while constructing this string,
* the result might be inconsistent. If someone has changed the state of node
* while we are constructing the string, the result will be inconsistent.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 5173b23..0d90728 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -266,11 +266,16 @@ public class MockNodeManager implements NodeManager {
/**
* Return the node stat of the specified datanode.
* @param datanodeDetails - datanode details.
- * @return node stat if it is live/stale, null if it is dead or does't exist.
+ * @return node stat if it is live/stale, null if it is decommissioned or
+ * doesn't exist.
*/
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) {
- return new SCMNodeMetric(nodeMetricMap.get(datanodeDetails.getUuid()));
+ SCMNodeStat stat = nodeMetricMap.get(datanodeDetails.getUuid());
+ if (stat == null) {
+ return null;
+ }
+ return new SCMNodeMetric(stat);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
index e50b823..f9b1392 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
@@ -57,9 +57,12 @@ public class TestNodeReportHandler implements EventPublisher {
StorageReportProto storageOne = TestUtils
.createStorageReport(dn.getUuid(), storagePath, 100, 10, 90, null);
+ SCMNodeMetric nodeMetric = nodeManager.getNodeStat(dn);
+ Assert.assertNull(nodeMetric);
+
nodeReportHandler.onMessage(
getNodeReport(dn, storageOne), this);
- SCMNodeMetric nodeMetric = nodeManager.getNodeStat(dn);
+ nodeMetric = nodeManager.getNodeStat(dn);
Assert.assertTrue(nodeMetric.get().getCapacity().get() == 100);
Assert.assertTrue(nodeMetric.get().getRemaining().get() == 90);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c798c1e/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 4238703..a9a00ef 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -196,7 +196,8 @@ public class ReplicationNodeManagerMock implements NodeManager {
* Return the node stat of the specified datanode.
*
* @param dd - datanode details.
- * @return node stat if it is live/stale, null if it is dead or does't exist.
+ * @return node stat if it is live/stale, null if it is decommissioned or
+ * doesn't exist.
*/
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails dd) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDDS-560. Create Generic exception
class to be used by S3 rest services. Contributed by Bharat Viswanadham.
Posted by in...@apache.org.
HDDS-560. Create Generic exception class to be used by S3 rest services. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81072d5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81072d5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81072d5e
Branch: refs/heads/HDFS-13532
Commit: 81072d5e3defad5ebbbd174fcfb15de0943cf010
Parents: 5edb9d3
Author: Márton Elek <el...@apache.org>
Authored: Tue Oct 2 10:00:10 2018 -0400
Committer: Márton Elek <el...@apache.org>
Committed: Tue Oct 2 10:00:10 2018 -0400
----------------------------------------------------------------------
hadoop-ozone/s3gateway/pom.xml | 5 +
.../apache/hadoop/ozone/s3/EndpointBase.java | 19 ++-
.../hadoop/ozone/s3/exception/OS3Exception.java | 161 +++++++++++++++++++
.../ozone/s3/exception/OS3ExceptionMapper.java | 41 +++++
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 81 ++++++++++
.../hadoop/ozone/s3/exception/package-info.java | 21 +++
.../hadoop/ozone/s3/object/ListObject.java | 3 +-
.../hadoop/ozone/s3/bucket/TestGetBucket.java | 7 +-
.../ozone/s3/exception/TestOS3Exception.java | 49 ++++++
.../hadoop/ozone/s3/exception/package-info.java | 22 +++
10 files changed, 401 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index 7b02c85..5eb0232 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -63,6 +63,11 @@
<version>2.27</version>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-xml</artifactId>
+ <version>2.9.0</version>
+ </dependency>
+ <dependency>
<groupId>javax.enterprise</groupId>
<artifactId>cdi-api</artifactId>
<version>1.2</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java
index bfbeeec..f20d182 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java
@@ -26,12 +26,20 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneVolume;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable.Resource;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Basic helpers for all the REST endpoints.
*/
public class EndpointBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(EndpointBase.class);
@Inject
private OzoneClient client;
@@ -41,13 +49,16 @@ public class EndpointBase {
}
protected OzoneBucket getBucket(OzoneVolume volume, String bucketName)
- throws IOException {
- OzoneBucket bucket = null;
+ throws OS3Exception, IOException {
+ OzoneBucket bucket;
try {
bucket = volume.getBucket(bucketName);
- } catch (Exception ex) {
+ } catch (IOException ex) {
+ LOG.error("Error occurred is {}", ex);
if (ex.getMessage().contains("NOT_FOUND")) {
- throw new NotFoundException("Bucket" + bucketName + " is not found");
+ OS3Exception oex = S3ErrorTable.newError(S3ErrorTable.NO_SUCH_BUCKET,
+ OzoneUtils.getRequestID(), Resource.BUCKET);
+ throw oex;
} else {
throw ex;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
new file mode 100644
index 0000000..722a4a1
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.exception;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.dataformat.xml.XmlMapper;
+import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.XmlRootElement;
+
+
+/**
+ * This class represents exceptions raised from Ozone S3 service.
+ *
+ * Ref:https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+ */
+@XmlRootElement(name = "Error")
+@XmlAccessorType(XmlAccessType.NONE)
+public class OS3Exception extends Exception {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OS3Exception.class);
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new XmlMapper();
+ mapper.registerModule(new JaxbAnnotationModule());
+ mapper.enable(SerializationFeature.INDENT_OUTPUT);
+ }
+ @XmlElement(name = "Code")
+ private String code;
+
+ @XmlElement(name = "Message")
+ private String errorMessage;
+
+ @XmlElement(name = "Resource")
+ private String resource;
+
+ @XmlElement(name = "RequestId")
+ private String requestId;
+
+ @XmlTransient
+ private int httpCode;
+
+ public OS3Exception() {
+ //Added for JaxB.
+ }
+
+ /**
+ * Create an object OS3Exception.
+ * @param codeVal
+ * @param messageVal
+ * @param requestIdVal
+ * @param resourceVal
+ */
+ public OS3Exception(String codeVal, String messageVal, String requestIdVal,
+ String resourceVal) {
+ this.code = codeVal;
+ this.errorMessage = messageVal;
+ this.requestId = requestIdVal;
+ this.resource = resourceVal;
+ }
+
+ /**
+ * Create an object OS3Exception.
+ * @param codeVal
+ * @param messageVal
+ * @param httpCode
+ */
+ public OS3Exception(String codeVal, String messageVal, int httpCode) {
+ this.code = codeVal;
+ this.errorMessage = messageVal;
+ this.httpCode = httpCode;
+ }
+
+ public String getCode() {
+ return code;
+ }
+
+ public void setCode(String code) {
+ this.code = code;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+ public void setErrorMessage(String errorMessage) {
+ this.errorMessage = errorMessage;
+ }
+
+ public String getRequestId() {
+ return requestId;
+ }
+
+ public void setRequestId(String requestId) {
+ this.requestId = requestId;
+ }
+
+ public String getResource() {
+ return resource;
+ }
+
+ public void setResource(String resource) {
+ this.resource = resource;
+ }
+
+ public int getHttpCode() {
+ return httpCode;
+ }
+
+ public void setHttpCode(int httpCode) {
+ this.httpCode = httpCode;
+ }
+
+ public String toXml() {
+ try {
+ String val = mapper.writeValueAsString(this);
+ LOG.debug("toXml val is {}", val);
+ String xmlLine = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + val;
+ return xmlLine;
+ } catch (Exception ex) {
+ LOG.error("Exception occurred {}", ex);
+ }
+
+ //When we get exception log it, and return exception as xml from actual
+ // exception data. So, falling back to construct from exception.
+ String formatString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +
+ "<Error>" +
+ "<Code>%s</Code>" +
+ "<Message>%s</Message>" +
+ "<Resource>%s</Resource>" +
+ "<RequestId>%s</RequestId>" +
+ "</Error>";
+ return String.format(formatString, this.getCode(),
+ this.getErrorMessage(), this.getResource(),
+ this.getRequestId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3ExceptionMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3ExceptionMapper.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3ExceptionMapper.java
new file mode 100644
index 0000000..3f39d72
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3ExceptionMapper.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.exception;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+/**
+ * Class the represents various errors returned by the
+ * Ozone S3 service.
+ */
+@Provider
+public class OS3ExceptionMapper implements ExceptionMapper<OS3Exception> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OS3ExceptionMapper.class);
+ @Override
+ public Response toResponse(OS3Exception exception) {
+ LOG.debug("Returning exception. ex: {}", exception.toString());
+ return Response.status(exception.getHttpCode())
+ .entity(exception.toXml()).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
new file mode 100644
index 0000000..b504bf4
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3.exception;
+
+
+import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+
+/**
+ * This class represents errors from Ozone S3 service.
+ * This class needs to be updated to add new errors when required.
+ */
+public final class S3ErrorTable {
+
+ private S3ErrorTable() {
+ //No one should construct this object.
+ }
+
+ public static final OS3Exception INVALID_URI = new OS3Exception("InvalidURI",
+ "Couldn't parse the specified URI.", HTTP_BAD_REQUEST);
+
+ public static final OS3Exception NO_SUCH_BUCKET = new OS3Exception(
+ "NoSuchBucket", "The specified bucket does not exist", HTTP_NOT_FOUND);
+
+
+ /**
+ * Create a new instance of Error.
+ * @param e Error Template
+ * @param requestID
+ * @param resource Resource associated with this exception
+ * @return creates a new instance of error based on the template
+ */
+ public static OS3Exception newError(OS3Exception e, String requestID,
+ Resource resource){
+ OS3Exception err = new OS3Exception(e.getCode(), e.getErrorMessage(),
+ e.getHttpCode());
+ err.setRequestId(requestID);
+ err.setResource(resource.getResource());
+ return err;
+ }
+
+ /**
+ * Resources, which can be defined in OS3Exception.
+ */
+ public enum Resource {
+ BUCKET("Bucket");
+
+ private final String resource;
+
+ /**
+ * Constructs resource.
+ * @param value
+ */
+ Resource(String value) {
+ this.resource = value;
+ }
+
+ /**
+ * Get resource.
+ * @return string
+ */
+ public String getResource() {
+ return this.resource;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/package-info.java
new file mode 100644
index 0000000..d295ae8
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains Ozone S3 exceptions.
+ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java
index 73f1343..a7bd7ad 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.s3.EndpointBase;
import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
/**
* List Object Rest endpoint.
@@ -55,7 +56,7 @@ public class ListObject extends EndpointBase {
@QueryParam("marker") String marker,
@DefaultValue("1000") @QueryParam("max-keys") int maxKeys,
@QueryParam("prefix") String prefix,
- @Context HttpHeaders hh) throws IOException {
+ @Context HttpHeaders hh) throws OS3Exception, IOException {
if (delimiter == null) {
delimiter = "/";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java
index f5a6f7e..123dd79 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.object.ListObject;
import org.apache.hadoop.ozone.s3.object.ListObjectResponse;
@@ -36,7 +37,7 @@ import org.junit.Test;
public class TestGetBucket {
@Test
- public void listRoot() throws IOException {
+ public void listRoot() throws OS3Exception, IOException {
ListObject getBucket = new ListObject();
@@ -58,7 +59,7 @@ public class TestGetBucket {
}
@Test
- public void listDir() throws IOException {
+ public void listDir() throws OS3Exception, IOException {
ListObject getBucket = new ListObject();
@@ -78,7 +79,7 @@ public class TestGetBucket {
}
@Test
- public void listSubDir() throws IOException {
+ public void listSubDir() throws OS3Exception, IOException {
ListObject getBucket = new ListObject();
OzoneClient ozoneClient =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exception.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exception.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exception.java
new file mode 100644
index 0000000..72e2949
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/TestOS3Exception.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.exception;
+
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests OS3Exception class.
+ */
+public class TestOS3Exception {
+
+ @Test
+ public void testOS3Exception() {
+ OS3Exception ex = new OS3Exception("AccessDenied", "Access Denied",
+ 403);
+ String requestId = OzoneUtils.getRequestID();
+ ex = S3ErrorTable.newError(ex, requestId, S3ErrorTable.Resource.BUCKET);
+ String val = ex.toXml();
+ String formatString = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<Error>\n" +
+ " <Code>%s</Code>\n" +
+ " <Message>%s</Message>\n" +
+ " <Resource>%s</Resource>\n" +
+ " <RequestId>%s</RequestId>\n" +
+ "</Error>\n";
+ String expected = String.format(formatString, ex.getCode(),
+ ex.getErrorMessage(), ex.getResource(),
+ ex.getRequestId());
+ Assert.assertEquals(expected, val);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81072d5e/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/package-info.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/package-info.java
new file mode 100644
index 0000000..31effe4
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/exception/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package tests OS3Exception.
+ */
+package org.apache.hadoop.ozone.s3.exception;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HADOOP-15793. ABFS: Skip
unsupported test cases when non namespace enabled in
ITestAzureBlobFileSystemAuthorization Contributed by Yuan Gao.
Posted by in...@apache.org.
HADOOP-15793. ABFS: Skip unsupported test cases when non namespace enabled in ITestAzureBlobFileSystemAuthorization
Contributed by Yuan Gao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43bc9848
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43bc9848
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43bc9848
Branch: refs/heads/HDFS-13532
Commit: 43bc9848914cee5e03a9965eb2165f42efb2c35d
Parents: 5689355
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 2 11:37:28 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 2 11:37:28 2018 +0100
----------------------------------------------------------------------
.../ITestAzureBlobFileSystemAuthorization.java | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43bc9848/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
index 37ab825..1705516 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java
@@ -35,6 +35,7 @@ import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry;
import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
import static org.apache.hadoop.fs.permission.AclEntryType.GROUP;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assume.assumeTrue;
/**
* Test Perform Authorization Check operation
@@ -202,6 +203,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetOwnerAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.setOwner(TEST_WRITE_ONLY_FILE_PATH_0, "testUser", "testGroup");
}
@@ -209,6 +211,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetOwnerUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -219,6 +222,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetPermissionAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.setPermission(TEST_WRITE_ONLY_FILE_PATH_0, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
}
@@ -226,6 +230,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetPermissionUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -236,6 +241,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testModifyAclEntriesAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
fs.modifyAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
@@ -244,6 +250,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testModifyAclEntriesUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
@@ -255,15 +262,16 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveAclEntriesAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
- //fs.modifyAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
fs.removeAclEntries(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
}
@Test
public void testRemoveAclEntriesUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
@@ -275,6 +283,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveDefaultAclAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.removeDefaultAcl(TEST_WRITE_ONLY_FILE_PATH_0);
}
@@ -282,6 +291,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveDefaultAclUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -292,6 +302,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveAclAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
fs.removeAcl(TEST_WRITE_ONLY_FILE_PATH_0);
}
@@ -299,6 +310,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testRemoveAclUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
intercept(AbfsAuthorizationException.class,
()-> {
@@ -309,6 +321,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetAclAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
fs.setAcl(TEST_WRITE_ONLY_FILE_PATH_0, aclSpec);
@@ -317,6 +330,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testSetAclUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
@@ -328,6 +342,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testGetAclStatusAuthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_THEN_READ_ONLY_PATH).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
fs.getAclStatus(TEST_WRITE_THEN_READ_ONLY_PATH);
@@ -336,6 +351,7 @@ public class ITestAzureBlobFileSystemAuthorization extends AbstractAbfsIntegrati
@Test
public void testGetAclStatusUnauthorized() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
+ assumeTrue("This test case only runs when namespace is enabled", fs.getIsNamespaceEnabeld());
fs.create(TEST_WRITE_ONLY_FILE_PATH_0).close();
List<AclEntry> aclSpec = Arrays.asList(aclEntry(ACCESS, GROUP, "bar", FsAction.ALL));
intercept(AbfsAuthorizationException.class,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-525. Support virtual-hosted
style URLs. Contributed by Bharat Viswanadham
Posted by in...@apache.org.
HDDS-525. Support virtual-hosted style URLs. Contributed by Bharat Viswanadham
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4eff629a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4eff629a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4eff629a
Branch: refs/heads/HDFS-13532
Commit: 4eff629ab3a330e8f1efe92857e76235cb412ef4
Parents: 59d5af2
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Oct 1 13:16:08 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Oct 1 13:16:08 2018 -0700
----------------------------------------------------------------------
.../common/src/main/resources/ozone-default.xml | 78 ++++++++-
hadoop-ozone/integration-test/pom.xml | 4 +
.../ozone/TestOzoneConfigurationFields.java | 4 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 1 +
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 143 ++++++++++++++++
.../ozone/s3/TestVirtualHostStyleFilter.java | 163 +++++++++++++++++++
.../apache/hadoop/ozone/s3/package-info.java | 21 +++
7 files changed, 411 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eff629a/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index f7681e8..b7c967d 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1035,12 +1035,12 @@
<property>
<name>hadoop.tags.custom</name>
- <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
+ <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY</value>
</property>
<property>
<name>ozone.tags.system</name>
- <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
+ <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY</value>
</property>
@@ -1222,4 +1222,78 @@
</description>
</property>
+ <property>
+ <name>ozone.s3g.authentication.kerberos.principal</name>
+ <value/>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>The server principal used by Ozone S3Gateway server. This is
+ typically set to
+ HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
+ HTTP/ by convention.</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.domain.name</name>
+ <value/>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>List of Ozone S3Gateway domain names. If multiple
+ domain names to be provided, they should be a "," seperated.
+ This parameter is only required when virtual host style pattern is
+ followed.</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.http-address</name>
+ <value>0.0.0.0:9878</value>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>The address and the base port where the Ozone S3Gateway
+ Server will
+ listen on.</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.http-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>The actual address the HTTP server will bind to. If this optional address
+ is set, it overrides only the hostname portion of ozone.s3g.http-address.
+ This is useful for making the Ozone S3Gateway HTTP server listen on all
+ interfaces by setting it to 0.0.0.0.</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.http.enabled</name>
+ <value>true</value>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>The boolean which enables the Ozone S3Gateway server
+ .</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.https-address</name>
+ <value/>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>Ozone S3Gateway serverHTTPS server address and port
+ .</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.https-bind-host</name>
+ <value/>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>The actual address the HTTPS server will bind to. If this optional address
+ is set, it overrides only the hostname portion of ozone.s3g.https-address.
+ This is useful for making the Ozone S3Gateway HTTPS server listen on all
+ interfaces by setting it to 0.0.0.0.</description>
+ </property>
+
+ <property>
+ <name>ozone.s3g.keytab.file</name>
+ <value/>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>The keytab file used by the S3Gateway server to login as its
+ service principal. </description>
+ </property>
+
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eff629a/hadoop-ozone/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml
index bed2fce..84e86d8 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -44,6 +44,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-s3gateway</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eff629a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 909cddf..986618c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.TestConfigurationFieldsBase;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
/**
* Tests if configuration constants documented in ozone-defaults.xml.
@@ -32,7 +33,8 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
xmlFilename = new String("ozone-default.xml");
configurationClasses =
new Class[] {OzoneConfigKeys.class, ScmConfigKeys.class,
- OMConfigKeys.class, HddsConfigKeys.class};
+ OMConfigKeys.class, HddsConfigKeys.class,
+ S3GatewayConfigKeys.class};
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
xmlPropsToSkipCompare.add("hadoop.tags.custom");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eff629a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
index c340a50..4a5570a 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
@@ -44,6 +44,7 @@ public final class S3GatewayConfigKeys {
public static final int OZONE_S3G_HTTPS_BIND_PORT_DEFAULT = 9879;
public static final String OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
"ozone.s3g.authentication.kerberos.principal";
+ public static final String OZONE_S3G_DOMAIN_NAME = "ozone.s3g.domain.name";
/**
* Never constructed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eff629a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
new file mode 100644
index 0000000..3bd690b
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3;
+
+import javax.inject.Inject;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.ext.Provider;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NAME;
+
+/**
+ * Filter used to convert virtual host style pattern to path style pattern.
+ */
+
+@Provider
+@PreMatching
+public class VirtualHostStyleFilter implements ContainerRequestFilter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ VirtualHostStyleFilter.class);
+ private static final Pattern URL_SCHEME_PATTERN = Pattern.compile("" +
+ "(?<bucket>(.+))\\.(?<volume>(.+))\\.");
+
+ @Inject
+ private OzoneConfiguration conf;
+ private String[] domains;
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws
+ IOException {
+ domains = conf.getTrimmedStrings(OZONE_S3G_DOMAIN_NAME);
+
+ if (domains.length == 0) {
+ // domains is not configured, might be it is path style.
+ // So, do not continue further, just return.
+ return;
+ }
+ //Get the value of the host
+ String host = requestContext.getHeaderString(HttpHeaders.HOST);
+ String domain = getDomainName(host);
+
+ if (domain == null) {
+ throw getException("Invalid S3 Gateway request {" + requestContext
+ .getUriInfo().getRequestUri().toString() + " }: No matching domain " +
+ "{" + Arrays.toString(domains) + "} for the host {" + host + "}");
+ }
+
+ LOG.debug("Http header host name is {}", host);
+ LOG.debug("Domain name matched is {}", domain);
+
+ //Check if we have a Virtual Host style request, host length greater than
+ // address length means it is virtual host style, we need to convert to
+ // path style.
+ if (host.length() > domain.length()) {
+ String bothNames = host.substring(0, host.length() - domain.length());
+ LOG.debug("Both volume name and bucket name is {}", bothNames);
+ Matcher matcher = URL_SCHEME_PATTERN.matcher(bothNames);
+
+ if (!matcher.matches()) {
+ throw getException("Invalid S3 Gateway request {" + requestContext
+ .getUriInfo().getRequestUri().toString() +"}:" +" Host: {" + host
+ + " is in invalid format");
+ }
+
+ String bucketStr = matcher.group("bucket");
+ String volumeStr = matcher.group("volume");
+
+ LOG.debug("bucket {}, volumeStr {}", bucketStr, volumeStr);
+
+ URI baseURI = requestContext.getUriInfo().getBaseUri();
+ String currentPath = requestContext.getUriInfo().getPath();
+ String newPath = String.format("%s/%s", volumeStr, bucketStr);
+ if (currentPath != null) {
+ newPath += String.format("%s", currentPath);
+ }
+ URI requestAddr = UriBuilder.fromUri(baseURI).path(newPath).build();
+ requestContext.setRequestUri(baseURI, requestAddr);
+ }
+ }
+
+ private InvalidRequestException getException(String message) {
+ return new InvalidRequestException(message);
+ }
+
+ @VisibleForTesting
+ public void setConfiguration(OzoneConfiguration config) {
+ this.conf = config;
+ }
+
+
+ /**
+ * This method finds the longest match with the domain name.
+ * @param host
+ * @return domain name matched with the host. if none of them are matching,
+ * return null.
+ */
+ private String getDomainName(String host) {
+ String match = null;
+ int length=0;
+ for (String domainVal : domains) {
+ if (host.endsWith(domainVal)) {
+ int len = domainVal.length();
+ if (len > length) {
+ length = len;
+ match = domainVal;
+ }
+ }
+ }
+ return match;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eff629a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
new file mode 100644
index 0000000..ac8fa87
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3;
+
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.glassfish.jersey.internal.PropertiesDelegate;
+import org.glassfish.jersey.server.ContainerRequest;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.SecurityContext;
+import java.net.URI;
+
+/**
+ * This class test virtual host style mapping conversion to path style.
+ */
+public class TestVirtualHostStyleFilter {
+
+ private static OzoneConfiguration conf;
+ private static String s3HttpAddr;
+
+ @Before
+ public void setup() {
+ conf = new OzoneConfiguration();
+ s3HttpAddr = "localhost:9878";
+ conf.set(S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY, s3HttpAddr);
+ conf.set(S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NAME, s3HttpAddr);
+ }
+
+ /**
+ * Create containerRequest object.
+ * @return ContainerRequest
+ * @throws Exception
+ */
+ public ContainerRequest createContainerRequest(String host, String path,
+ boolean virtualHostStyle)
+ throws Exception {
+ URI baseUri = new URI("http://" + s3HttpAddr);
+ URI virtualHostStyleUri;
+ if (path == null) {
+ virtualHostStyleUri = new URI("http://" + s3HttpAddr);
+ } else {
+ virtualHostStyleUri = new URI("http://" + s3HttpAddr + path);
+ }
+ URI pathStyleUri = new URI("http://" + s3HttpAddr + path);
+ String httpMethod = "DELETE";
+ SecurityContext securityContext = Mockito.mock(SecurityContext.class);
+ PropertiesDelegate propertiesDelegate = Mockito.mock(PropertiesDelegate
+ .class);
+ ContainerRequest containerRequest;
+ if (virtualHostStyle) {
+ containerRequest = new ContainerRequest(baseUri, virtualHostStyleUri,
+ httpMethod, securityContext, propertiesDelegate);
+ containerRequest.header(HttpHeaders.HOST, host);
+ } else {
+ containerRequest = new ContainerRequest(baseUri, pathStyleUri,
+ httpMethod, securityContext, propertiesDelegate);
+ containerRequest.header(HttpHeaders.HOST, host);
+ }
+ return containerRequest;
+ }
+
+ @Test
+ public void testVirtualHostStyle() throws Exception {
+ VirtualHostStyleFilter virtualHostStyleFilter =
+ new VirtualHostStyleFilter();
+ virtualHostStyleFilter.setConfiguration(conf);
+
+ ContainerRequest containerRequest = createContainerRequest("mybucket" +
+ ".myvolume.localhost:9878", "/myfile", true);
+ virtualHostStyleFilter.filter(containerRequest);
+ URI expected = new URI("http://" + s3HttpAddr +
+ "/myvolume/mybucket/myfile");
+ Assert.assertEquals(expected, containerRequest.getRequestUri());
+ }
+
+ @Test
+ public void testPathStyle() throws Exception {
+
+ VirtualHostStyleFilter virtualHostStyleFilter =
+ new VirtualHostStyleFilter();
+ virtualHostStyleFilter.setConfiguration(conf);
+
+ ContainerRequest containerRequest = createContainerRequest(s3HttpAddr,
+ "/myvolume/mybucket/myfile", false);
+ virtualHostStyleFilter.filter(containerRequest);
+ URI expected = new URI("http://" + s3HttpAddr +
+ "/myvolume/mybucket/myfile");
+ Assert.assertEquals(expected, containerRequest.getRequestUri());
+
+ }
+
+ @Test
+ public void testVirtualHostStyleWithCreateBucketRequest() throws Exception {
+
+ VirtualHostStyleFilter virtualHostStyleFilter =
+ new VirtualHostStyleFilter();
+ virtualHostStyleFilter.setConfiguration(conf);
+
+ ContainerRequest containerRequest = createContainerRequest("mybucket" +
+ ".myvolume.localhost:9878", null, true);
+ virtualHostStyleFilter.filter(containerRequest);
+ URI expected = new URI("http://" + s3HttpAddr +
+ "/myvolume/mybucket");
+ Assert.assertEquals(expected, containerRequest.getRequestUri());
+
+ }
+
+ @Test
+ public void testVirtualHostStyleWithNoMatchingDomain() throws Exception {
+
+ VirtualHostStyleFilter virtualHostStyleFilter =
+ new VirtualHostStyleFilter();
+ virtualHostStyleFilter.setConfiguration(conf);
+
+ ContainerRequest containerRequest = createContainerRequest("mybucket" +
+ ".myvolume.localhost:9999", null, true);
+ try {
+ virtualHostStyleFilter.filter(containerRequest);
+ } catch (InvalidRequestException ex) {
+ GenericTestUtils.assertExceptionContains("No matching domain", ex);
+ }
+
+ }
+
+ @Test
+ public void testVirtualHostStyleWithoutVolumeName() throws Exception {
+
+ VirtualHostStyleFilter virtualHostStyleFilter =
+ new VirtualHostStyleFilter();
+ virtualHostStyleFilter.setConfiguration(conf);
+
+ ContainerRequest containerRequest = createContainerRequest("mybucket." +
+ ".localhost:9878", null, true);
+ try {
+ virtualHostStyleFilter.filter(containerRequest);
+ } catch (InvalidRequestException ex) {
+ GenericTestUtils.assertExceptionContains("invalid format", ex);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eff629a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/package-info.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/package-info.java
new file mode 100644
index 0000000..e7e04ab
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Unit tests for the bucket related rest endpoints.
+ */
+package org.apache.hadoop.ozone.s3;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-8621. Add test coverage of
custom Resource Types for the apps/ REST API endpoint. (Contributed by
Szilard Nemeth)
Posted by in...@apache.org.
YARN-8621. Add test coverage of custom Resource Types for the apps/<appId> REST API endpoint. (Contributed by Szilard Nemeth)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0ee6fbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0ee6fbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0ee6fbe
Branch: refs/heads/HDFS-13532
Commit: d0ee6fbe281b9edfab5913ca46a0f89ee9a2a6cc
Parents: 4eff629
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Oct 1 14:45:51 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Mon Oct 1 14:46:42 2018 -0700
----------------------------------------------------------------------
.../webapp/TestRMWebServicesAppAttempts.java | 397 +++++++++++++++++++
...TestRMWebServicesAppCustomResourceTypes.java | 170 ++++++++
.../webapp/TestRMWebServicesApps.java | 287 --------------
...estRMWebServicesAppsCustomResourceTypes.java | 81 +---
...RMWebServicesCustomResourceTypesCommons.java | 99 +++++
.../webapp/helper/BufferedClientResponse.java | 9 +
.../helper/JsonCustomResourceTypeTestcase.java | 9 +
.../helper/XmlCustomResourceTypeTestCase.java | 9 +
8 files changed, 702 insertions(+), 359 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java
new file mode 100644
index 0000000..a84fdca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.util.Collection;
+
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestRMWebServicesAppAttempts extends JerseyTestBase {
+
+ private static MockRM rm;
+
+ private static final int CONTAINER_MB = 1024;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+ ResourceScheduler.class);
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ }
+ }
+
+ static {
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
+ }
+
+ public TestRMWebServicesAppAttempts() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testAppAttempts() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+ amNodeManager.nodeHeartbeat(true);
+ testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
+ MediaType.APPLICATION_JSON);
+ rm.stop();
+ }
+
+ @Test (timeout = 20000)
+ public void testMultipleAppAttempts() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+ MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+ int maxAppAttempts = rm.getConfig().getInt(
+ YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
+ assertTrue(maxAppAttempts > 1);
+ int numAttempt = 1;
+ while (true) {
+ // fail the AM by sending CONTAINER_FINISHED event without registering.
+ amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
+ ContainerState.COMPLETE);
+ rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
+ if (numAttempt == maxAppAttempts) {
+ rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
+ break;
+ }
+ // wait for app to start a new attempt.
+ rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
+ am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+ numAttempt++;
+ }
+ assertEquals("incorrect number of attempts", maxAppAttempts,
+ app1.getAppAttempts().values().size());
+ testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
+ MediaType.APPLICATION_JSON);
+ rm.stop();
+ }
+
+ @Test
+ public void testAppAttemptsSlash() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
+ amNodeManager.nodeHeartbeat(true);
+ testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1,
+ MediaType.APPLICATION_JSON);
+ rm.stop();
+ }
+
+ @Test
+ public void testAppAttemptsDefault() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB);
+ amNodeManager.nodeHeartbeat(true);
+ testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, "");
+ rm.stop();
+ }
+
+ @Test
+ public void testInvalidAppIdGetAttempts() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app = rm.submitApp(CONTAINER_MB);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .path("application_invalid_12").path("appattempts")
+ .accept(MediaType.APPLICATION_JSON)
+ .get(JSONObject.class);
+ fail("should have thrown exception on invalid appAttempt");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+
+ assertResponseStatusCode(ClientResponse.Status.BAD_REQUEST,
+ response.getStatusInfo());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ checkStringMatch("exception message",
+ "java.lang.IllegalArgumentException: Invalid ApplicationId:"
+ + " application_invalid_12",
+ message);
+ checkStringMatch("exception type",
+ "BadRequestException", type);
+ checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
+
+ } finally {
+ rm.stop();
+ }
+ }
+
+ @Test
+ public void testInvalidAppAttemptId() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app = rm.submitApp(CONTAINER_MB);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .path(app.getApplicationId().toString()).path("appattempts")
+ .path("appattempt_invalid_12_000001")
+ .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
+ fail("should have thrown exception on invalid appAttempt");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+
+ assertResponseStatusCode(ClientResponse.Status.BAD_REQUEST,
+ response.getStatusInfo());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ checkStringMatch("exception message",
+ "java.lang.IllegalArgumentException: Invalid AppAttemptId:"
+ + " appattempt_invalid_12_000001",
+ message);
+ checkStringMatch("exception type",
+ "BadRequestException", type);
+ checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
+
+ } finally {
+ rm.stop();
+ }
+ }
+
+ @Test
+ public void testNonexistAppAttempts() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .path("application_00000_0099").accept(MediaType.APPLICATION_JSON)
+ .get(JSONObject.class);
+ fail("should have thrown exception on invalid appid");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+
+ assertResponseStatusCode(ClientResponse.Status.NOT_FOUND,
+ response.getStatusInfo());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ checkStringMatch("exception message",
+ "java.lang.Exception: app with id: application_00000_0099 not found",
+ message);
+ checkStringMatch("exception type",
+ "NotFoundException", type);
+ checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ } finally {
+ rm.stop();
+ }
+ }
+
+ private void testAppAttemptsHelper(String path, RMApp app, String media)
+ throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").path(path).path("appattempts").accept(media)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject jsonAppAttempts = json.getJSONObject("appAttempts");
+ assertEquals("incorrect number of elements", 1, jsonAppAttempts.length());
+ JSONArray jsonArray = jsonAppAttempts.getJSONArray("appAttempt");
+
+ Collection<RMAppAttempt> attempts = app.getAppAttempts().values();
+ assertEquals("incorrect number of elements", attempts.size(),
+ jsonArray.length());
+
+ // Verify these parallel arrays are the same
+ int i = 0;
+ for (RMAppAttempt attempt : attempts) {
+ verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt, app.getUser());
+ ++i;
+ }
+ }
+
+ @Test
+ public void testAppAttemptsXML() throws Exception {
+ rm.start();
+ String user = "user1";
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").path(app1.getApplicationId().toString())
+ .path("appattempts").accept(MediaType.APPLICATION_XML)
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ String xml = response.getEntity(String.class);
+
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("appAttempts");
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+ NodeList attempt = dom.getElementsByTagName("appAttempt");
+ assertEquals("incorrect number of elements", 1, attempt.getLength());
+ verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt(), user);
+ rm.stop();
+ }
+
+ private void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
+ String user) {
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+
+ verifyAppAttemptInfoGeneric(appAttempt,
+ WebServicesTestUtils.getXmlInt(element, "id"),
+ WebServicesTestUtils.getXmlLong(element, "startTime"),
+ WebServicesTestUtils.getXmlString(element, "containerId"),
+ WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
+ WebServicesTestUtils.getXmlString(element, "nodeId"),
+ WebServicesTestUtils.getXmlString(element, "logsLink"), user);
+ }
+ }
+
+ private void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt,
+ String user)
+ throws Exception {
+
+ assertEquals("incorrect number of elements", 10, info.length());
+
+ verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
+ info.getLong("startTime"), info.getString("containerId"),
+ info.getString("nodeHttpAddress"), info.getString("nodeId"),
+ info.getString("logsLink"), user);
+ }
+
+ private void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
+ long startTime, String containerId, String nodeHttpAddress, String
+ nodeId,
+ String logsLink, String user) {
+
+ assertEquals("id doesn't match", appAttempt.getAppAttemptId()
+ .getAttemptId(), id);
+ assertEquals("startedTime doesn't match", appAttempt.getStartTime(),
+ startTime);
+ checkStringMatch("containerId", appAttempt
+ .getMasterContainer().getId().toString(), containerId);
+ checkStringMatch("nodeHttpAddress", appAttempt
+ .getMasterContainer().getNodeHttpAddress(), nodeHttpAddress);
+ checkStringMatch("nodeId", appAttempt
+ .getMasterContainer().getNodeId().toString(), nodeId);
+ assertTrue("logsLink doesn't match ", logsLink.startsWith("http://"));
+ assertTrue(
+ "logsLink doesn't contain user info", logsLink.endsWith("/"
+ + user));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java
new file mode 100644
index 0000000..ab4dcf8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppCustomResourceTypes.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.NodeList;
+
+import javax.ws.rs.core.MediaType;
+import java.util.ArrayList;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCustomResourceTypesCommons.verifyAppInfoJson;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCustomResourceTypesCommons.verifyAppsXML;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This test verifies that custom resource types are correctly serialized to XML
+ * and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps.
+ */
+public class TestRMWebServicesAppCustomResourceTypes extends JerseyTestBase {
+
+ private static MockRM rm;
+ private static final int CONTAINER_MB = 1024;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+ ResourceScheduler.class);
+ initResourceTypes(conf);
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ }
+
+ private void initResourceTypes(Configuration conf) {
+ conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ CustomResourceTypesConfigurationProvider.class.getName());
+ ResourceUtils.resetResourceTypes(conf);
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ createInjectorForWebServletModule();
+ }
+
+ private void createInjectorForWebServletModule() {
+ GuiceServletConfig
+ .setInjector(Guice.createInjector(new WebServletModule()));
+ }
+
+ public TestRMWebServicesAppCustomResourceTypes() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ private WebResource getWebResourcePathForApp(RMApp app1, WebResource r) {
+ return r.path("ws").path("v1").path("cluster").path("apps")
+ .path(String.valueOf(app1.getApplicationId().toString()));
+ }
+
+ @Test
+ public void testRunningAppXml() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+ am1.allocate("*", 2048, 1, new ArrayList<>());
+ amNodeManager.nodeHeartbeat(true);
+
+ WebResource r = resource();
+ WebResource path = getWebResourcePathForApp(app1, r);
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+ XmlCustomResourceTypeTestCase testCase =
+ new XmlCustomResourceTypeTestCase(path,
+ new BufferedClientResponse(response));
+ testCase.verify(document -> {
+ NodeList appArray = document
+ .getElementsByTagName("app");
+ assertEquals("incorrect number of app elements", 1, appArray.getLength());
+
+ verifyAppsXML(appArray, app1, rm);
+ });
+
+ rm.stop();
+ }
+
+ @Test
+ public void testRunningAppJson() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+ am1.allocate("*", 2048, 1, new ArrayList<>());
+ amNodeManager.nodeHeartbeat(true);
+
+ WebResource r = resource();
+ WebResource path = getWebResourcePathForApp(app1, r);
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+ JsonCustomResourceTypeTestcase testCase =
+ new JsonCustomResourceTypeTestcase(path,
+ new BufferedClientResponse(response));
+ testCase.verify(json -> {
+ try {
+ assertEquals("incorrect number of app elements", 1, json.length());
+ JSONObject app = json.getJSONObject("app");
+ verifyAppInfoJson(app, app1, rm);
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ rm.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 15f94e1..e9fd577 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1781,292 +1781,5 @@ public class TestRMWebServicesApps extends JerseyTestBase {
enforceExecutionType);
}
- @Test
- public void testAppAttempts() throws JSONException, Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- amNodeManager.nodeHeartbeat(true);
- testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
- MediaType.APPLICATION_JSON);
- rm.stop();
- }
-
- @Test (timeout = 20000)
- public void testMultipleAppAttempts() throws JSONException, Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
- RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
- int maxAppAttempts = rm.getConfig().getInt(
- YarnConfiguration.RM_AM_MAX_ATTEMPTS,
- YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
- assertTrue(maxAppAttempts > 1);
- int numAttempt = 1;
- while (true) {
- // fail the AM by sending CONTAINER_FINISHED event without registering.
- amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
- ContainerState.COMPLETE);
- rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
- if (numAttempt == maxAppAttempts) {
- rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
- break;
- }
- // wait for app to start a new attempt.
- rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
- am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
- numAttempt++;
- }
- assertEquals("incorrect number of attempts", maxAppAttempts,
- app1.getAppAttempts().values().size());
- testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
- MediaType.APPLICATION_JSON);
- rm.stop();
- }
-
- @Test
- public void testAppAttemptsSlash() throws JSONException, Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(CONTAINER_MB);
- amNodeManager.nodeHeartbeat(true);
- testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1,
- MediaType.APPLICATION_JSON);
- rm.stop();
- }
-
- @Test
- public void testAppAttemtpsDefault() throws JSONException, Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(CONTAINER_MB);
- amNodeManager.nodeHeartbeat(true);
- testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, "");
- rm.stop();
- }
-
- @Test
- public void testInvalidAppIdGetAttempts() throws JSONException, Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app = rm.submitApp(CONTAINER_MB);
- amNodeManager.nodeHeartbeat(true);
- WebResource r = resource();
-
- try {
- r.path("ws").path("v1").path("cluster").path("apps")
- .path("application_invalid_12").path("appattempts")
- .accept(MediaType.APPLICATION_JSON)
- .get(JSONObject.class);
- fail("should have thrown exception on invalid appAttempt");
- } catch (UniformInterfaceException ue) {
- ClientResponse response = ue.getResponse();
-
- assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- JSONObject msg = response.getEntity(JSONObject.class);
- JSONObject exception = msg.getJSONObject("RemoteException");
- assertEquals("incorrect number of elements", 3, exception.length());
- String message = exception.getString("message");
- String type = exception.getString("exception");
- String classname = exception.getString("javaClassName");
- WebServicesTestUtils.checkStringMatch("exception message",
- "java.lang.IllegalArgumentException: Invalid ApplicationId:"
- + " application_invalid_12",
- message);
- WebServicesTestUtils.checkStringMatch("exception type",
- "BadRequestException", type);
- WebServicesTestUtils.checkStringMatch("exception classname",
- "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
-
- } finally {
- rm.stop();
- }
- }
-
- @Test
- public void testInvalidAppAttemptId() throws JSONException, Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app = rm.submitApp(CONTAINER_MB);
- amNodeManager.nodeHeartbeat(true);
- WebResource r = resource();
-
- try {
- r.path("ws").path("v1").path("cluster").path("apps")
- .path(app.getApplicationId().toString()).path("appattempts")
- .path("appattempt_invalid_12_000001")
- .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
- fail("should have thrown exception on invalid appAttempt");
- } catch (UniformInterfaceException ue) {
- ClientResponse response = ue.getResponse();
-
- assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- JSONObject msg = response.getEntity(JSONObject.class);
- JSONObject exception = msg.getJSONObject("RemoteException");
- assertEquals("incorrect number of elements", 3, exception.length());
- String message = exception.getString("message");
- String type = exception.getString("exception");
- String classname = exception.getString("javaClassName");
- WebServicesTestUtils.checkStringMatch("exception message",
- "java.lang.IllegalArgumentException: Invalid AppAttemptId:"
- + " appattempt_invalid_12_000001",
- message);
- WebServicesTestUtils.checkStringMatch("exception type",
- "BadRequestException", type);
- WebServicesTestUtils.checkStringMatch("exception classname",
- "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
-
- } finally {
- rm.stop();
- }
- }
-
- @Test
- public void testNonexistAppAttempts() throws JSONException, Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- amNodeManager.nodeHeartbeat(true);
- WebResource r = resource();
-
- try {
- r.path("ws").path("v1").path("cluster").path("apps")
- .path("application_00000_0099").accept(MediaType.APPLICATION_JSON)
- .get(JSONObject.class);
- fail("should have thrown exception on invalid appid");
- } catch (UniformInterfaceException ue) {
- ClientResponse response = ue.getResponse();
-
- assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo());
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
-
- JSONObject msg = response.getEntity(JSONObject.class);
- JSONObject exception = msg.getJSONObject("RemoteException");
- assertEquals("incorrect number of elements", 3, exception.length());
- String message = exception.getString("message");
- String type = exception.getString("exception");
- String classname = exception.getString("javaClassName");
- WebServicesTestUtils.checkStringMatch("exception message",
- "java.lang.Exception: app with id: application_00000_0099 not found",
- message);
- WebServicesTestUtils.checkStringMatch("exception type",
- "NotFoundException", type);
- WebServicesTestUtils.checkStringMatch("exception classname",
- "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
- } finally {
- rm.stop();
- }
- }
-
- public void testAppAttemptsHelper(String path, RMApp app, String media)
- throws JSONException, Exception {
- WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("apps").path(path).path("appattempts").accept(media)
- .get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- JSONObject json = response.getEntity(JSONObject.class);
- assertEquals("incorrect number of elements", 1, json.length());
- JSONObject jsonAppAttempts = json.getJSONObject("appAttempts");
- assertEquals("incorrect number of elements", 1, jsonAppAttempts.length());
- JSONArray jsonArray = jsonAppAttempts.getJSONArray("appAttempt");
-
- Collection<RMAppAttempt> attempts = app.getAppAttempts().values();
- assertEquals("incorrect number of elements", attempts.size(),
- jsonArray.length());
-
- // Verify these parallel arrays are the same
- int i = 0;
- for (RMAppAttempt attempt : attempts) {
- verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt, app.getUser());
- ++i;
- }
- }
-
- @Test
- public void testAppAttemptsXML() throws JSONException, Exception {
- rm.start();
- String user = "user1";
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user);
- amNodeManager.nodeHeartbeat(true);
- WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("apps").path(app1.getApplicationId().toString())
- .path("appattempts").accept(MediaType.APPLICATION_XML)
- .get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- String xml = response.getEntity(String.class);
-
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder db = dbf.newDocumentBuilder();
- InputSource is = new InputSource();
- is.setCharacterStream(new StringReader(xml));
- Document dom = db.parse(is);
- NodeList nodes = dom.getElementsByTagName("appAttempts");
- assertEquals("incorrect number of elements", 1, nodes.getLength());
- NodeList attempt = dom.getElementsByTagName("appAttempt");
- assertEquals("incorrect number of elements", 1, attempt.getLength());
- verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt(), user);
- rm.stop();
- }
-
- public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
- String user)
- throws JSONException, Exception {
-
- for (int i = 0; i < nodes.getLength(); i++) {
- Element element = (Element) nodes.item(i);
-
- verifyAppAttemptInfoGeneric(appAttempt,
- WebServicesTestUtils.getXmlInt(element, "id"),
- WebServicesTestUtils.getXmlLong(element, "startTime"),
- WebServicesTestUtils.getXmlString(element, "containerId"),
- WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
- WebServicesTestUtils.getXmlString(element, "nodeId"),
- WebServicesTestUtils.getXmlString(element, "logsLink"), user);
- }
- }
-
- public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt,
- String user)
- throws JSONException, Exception {
-
- assertEquals("incorrect number of elements", 10, info.length());
-
- verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
- info.getLong("startTime"), info.getString("containerId"),
- info.getString("nodeHttpAddress"), info.getString("nodeId"),
- info.getString("logsLink"), user);
- }
-
- public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
- long startTime, String containerId, String nodeHttpAddress, String nodeId,
- String logsLink, String user)
- throws JSONException, Exception {
-
- assertEquals("id doesn't match", appAttempt.getAppAttemptId()
- .getAttemptId(), id);
- assertEquals("startedTime doesn't match", appAttempt.getStartTime(),
- startTime);
- WebServicesTestUtils.checkStringMatch("containerId", appAttempt
- .getMasterContainer().getId().toString(), containerId);
- WebServicesTestUtils.checkStringMatch("nodeHttpAddress", appAttempt
- .getMasterContainer().getNodeHttpAddress(), nodeHttpAddress);
- WebServicesTestUtils.checkStringMatch("nodeId", appAttempt
- .getMasterContainer().getNodeId().toString(), nodeId);
- assertTrue("logsLink doesn't match ", logsLink.startsWith("http://"));
- assertTrue(
- "logsLink doesn't contain user info", logsLink.endsWith("/"
- + user));
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
index 83e0056..bcdb713 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
@@ -25,24 +25,18 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -54,12 +48,15 @@ import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Element;
-import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp
+ .TestRMWebServicesCustomResourceTypesCommons.verifyAppInfoJson;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp
+ .TestRMWebServicesCustomResourceTypesCommons.verifyAppsXML;
import static org.junit.Assert.assertEquals;
/**
@@ -116,7 +113,7 @@ public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
}
@Test
- public void testRunningAppXml() throws Exception {
+ public void testRunningAppsXml() throws Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
@@ -140,14 +137,14 @@ public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
.getElementsByTagName("app");
assertEquals("incorrect number of app elements", 1, appArray.getLength());
- verifyAppsXML(appArray, app1);
+ verifyAppsXML(appArray, app1, rm);
});
rm.stop();
}
@Test
- public void testRunningAppJson() throws Exception {
+ public void testRunningAppsJson() throws Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
@@ -171,7 +168,7 @@ public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect count of app", 1, array.length());
- verifyAppInfoJson(array.getJSONObject(0), app1);
+ verifyAppInfoJson(array.getJSONObject(0), app1, rm);
} catch (JSONException e) {
throw new RuntimeException(e);
}
@@ -179,64 +176,4 @@ public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
rm.stop();
}
-
- private void verifyAppsXML(NodeList appArray, RMApp app) {
- for (int i = 0; i < appArray.getLength(); i++) {
- Element element = (Element) appArray.item(i);
- AppInfoXmlVerifications.verify(element, app);
-
- NodeList resourceRequests =
- element.getElementsByTagName("resourceRequests");
- assertEquals(1, resourceRequests.getLength());
- Node resourceRequest = resourceRequests.item(0);
- ResourceRequest rr =
- ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
- .getApplicationAttempt(
- app.getCurrentAppAttempt().getAppAttemptId())
- .getAppSchedulingInfo().getAllResourceRequests().get(0);
- ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
- (Element) resourceRequest, rr,
- CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
- }
- }
-
- private void verifyAppInfoJson(JSONObject info, RMApp app) throws
- JSONException {
- int expectedNumberOfElements = getExpectedNumberOfElements(app);
-
- assertEquals("incorrect number of elements", expectedNumberOfElements,
- info.length());
-
- AppInfoJsonVerifications.verify(info, app);
-
- JSONArray resourceRequests = info.getJSONArray("resourceRequests");
- JSONObject requestInfo = resourceRequests.getJSONObject(0);
- ResourceRequest rr =
- ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
- .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
- .getAppSchedulingInfo().getAllResourceRequests().get(0);
-
- ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(
- requestInfo, rr,
- CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
- }
-
- private int getExpectedNumberOfElements(RMApp app) {
- int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
- if (app.getApplicationSubmissionContext()
- .getNodeLabelExpression() != null) {
- expectedNumberOfElements++;
- }
-
- if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
- expectedNumberOfElements++;
- }
-
- if (AppInfo
- .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
- expectedNumberOfElements++;
- }
- return expectedNumberOfElements;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCustomResourceTypesCommons.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCustomResourceTypesCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCustomResourceTypesCommons.java
new file mode 100644
index 0000000..63ec19c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCustomResourceTypesCommons.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+ .AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler
+ .CustomResourceTypesConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestRMWebServicesCustomResourceTypesCommons {
+
+ static void verifyAppsXML(NodeList appArray, RMApp app, ResourceManager rm) {
+ for (int i = 0; i < appArray.getLength(); i++) {
+ Element element = (Element) appArray.item(i);
+ AppInfoXmlVerifications.verify(element, app);
+
+ NodeList resourceRequests =
+ element.getElementsByTagName("resourceRequests");
+ assertEquals(1, resourceRequests.getLength());
+ Node resourceRequest = resourceRequests.item(0);
+ ResourceRequest rr =
+ ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+ .getApplicationAttempt(
+ app.getCurrentAppAttempt().getAppAttemptId())
+ .getAppSchedulingInfo().getAllResourceRequests().get(0);
+ ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
+ (Element) resourceRequest, rr,
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+ }
+
+ static void verifyAppInfoJson(JSONObject info, RMApp app, ResourceManager rm)
+ throws JSONException {
+ int expectedNumberOfElements = getExpectedNumberOfElements(app);
+
+ assertEquals("incorrect number of elements", expectedNumberOfElements,
+ info.length());
+
+ AppInfoJsonVerifications.verify(info, app);
+
+ JSONArray resourceRequests = info.getJSONArray("resourceRequests");
+ JSONObject requestInfo = resourceRequests.getJSONObject(0);
+ ResourceRequest rr =
+ ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+ .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
+ .getAppSchedulingInfo().getAllResourceRequests().get(0);
+
+ ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(requestInfo,
+ rr, CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+
+ static int getExpectedNumberOfElements(RMApp app) {
+ int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
+ if (app.getApplicationSubmissionContext()
+ .getNodeLabelExpression() != null) {
+ expectedNumberOfElements++;
+ }
+
+ if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
+ expectedNumberOfElements++;
+ }
+
+ if (AppInfo
+ .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
+ expectedNumberOfElements++;
+ }
+ return expectedNumberOfElements;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
index 92bccbf..8953293 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
@@ -24,6 +24,7 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import java.io.IOException;
/**
@@ -54,4 +55,12 @@ public class BufferedClientResponse {
public MediaType getType() {
return response.getType();
}
+
+ public int getStatus() {
+ return response.getStatus();
+ }
+
+ public Response.StatusType getStatusInfo() {
+ return response.getStatusInfo();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
index 9d6a111..2b8d08d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
@@ -50,10 +50,19 @@ public class JsonCustomResourceTypeTestcase {
public JsonCustomResourceTypeTestcase(WebResource path,
BufferedClientResponse response) {
this.path = path;
+ verifyStatus(response);
this.response = response;
this.parsedResponse = response.getEntity(JSONObject.class);
}
+ private void verifyStatus(BufferedClientResponse response) {
+ String responseStr = response.getEntity(String.class);
+ assertEquals("HTTP status should be 200, " +
+ "status info: " + response.getStatusInfo() +
+ " response as string: " + responseStr,
+ 200, response.getStatus());
+ }
+
public void verify(Consumer<JSONObject> verifier) {
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0ee6fbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
index 29260aa..0ad92d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
@@ -59,9 +59,18 @@ public class XmlCustomResourceTypeTestCase {
public XmlCustomResourceTypeTestCase(WebResource path,
BufferedClientResponse response) {
this.path = path;
+ verifyStatus(response);
this.response = response;
}
+ private void verifyStatus(BufferedClientResponse response) {
+ String responseStr = response.getEntity(String.class);
+ assertEquals("HTTP status should be 200, " +
+ "status info: " + response.getStatusInfo() +
+ " response as string: " + responseStr,
+ 200, response.getStatus());
+ }
+
public void verify(Consumer<Document> verifier) {
assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8,
response.getType().toString());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDDS-401. Update storage
statistics on dead node. Contributed by LiXin Ge.
Posted by in...@apache.org.
HDDS-401. Update storage statistics on dead node. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/184544ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/184544ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/184544ef
Branch: refs/heads/HDFS-13532
Commit: 184544eff8b3d5951e4f04d080fa00de8e1eec95
Parents: f9c0221
Author: Ajay Kumar <aj...@apache.com>
Authored: Thu Sep 27 09:56:30 2018 -0700
Committer: Ajay Kumar <aj...@apache.com>
Committed: Thu Sep 27 09:56:30 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 7 +-
.../hadoop/hdds/scm/node/NodeManager.java | 7 ++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 16 +++
.../scm/server/StorageContainerManager.java | 2 +-
.../hdds/scm/container/MockNodeManager.java | 15 +++
.../hdds/scm/node/TestDeadNodeHandler.java | 109 ++++++++++++++++---
.../testutils/ReplicationNodeManagerMock.java | 9 ++
7 files changed, 145 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
index 7fda67d..88f5df7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java
@@ -42,19 +42,24 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
private final ContainerStateManager containerStateManager;
+ private final NodeManager nodeManager;
+
private static final Logger LOG =
LoggerFactory.getLogger(DeadNodeHandler.class);
public DeadNodeHandler(
Node2ContainerMap node2ContainerMap,
- ContainerStateManager containerStateManager) {
+ ContainerStateManager containerStateManager, NodeManager nodeManager) {
this.node2ContainerMap = node2ContainerMap;
this.containerStateManager = containerStateManager;
+ this.nodeManager = nodeManager;
}
@Override
public void onMessage(DatanodeDetails datanodeDetails,
EventPublisher publisher) {
+ nodeManager.processDeadNode(datanodeDetails.getUuid());
+
Set<ContainerID> containers =
node2ContainerMap.getContainers(datanodeDetails.getUuid());
if (containers == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index deb1628..fc73253 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -147,4 +147,11 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* @param nodeReport
*/
void processNodeReport(UUID dnUuid, NodeReportProto nodeReport);
+
+ /**
+ * Process a dead node event in this Node Manager.
+ *
+ * @param dnUuid datanode uuid.
+ */
+ void processDeadNode(UUID dnUuid);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 58da1cc..fd5f5de 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -490,4 +490,20 @@ public class SCMNodeManager
addDatanodeCommand(commandForDatanode.getDatanodeId(),
commandForDatanode.getCommand());
}
+
+ /**
+ * Remove the node stats and update the storage stats
+ * in this SCM Node Manager.
+ *
+ * @param dnUuid datanode uuid.
+ */
+ @Override
+ public void processDeadNode(UUID dnUuid) {
+ SCMNodeStat stat = nodeStats.get(dnUuid);
+ LOG.trace("Update stat values as Datanode {} is dead.", dnUuid);
+ if (stat != null) {
+ scmStat.subtract(stat);
+ stat.set(0, 0, 0);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index f491d30..52af62d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -229,7 +229,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new StaleNodeHandler(node2ContainerMap,
scmContainerManager.getPipelineSelector());
DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap,
- getScmContainerManager().getStateManager());
+ getScmContainerManager().getStateManager(), scmNodeManager);
ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
PendingDeleteHandler pendingDeleteHandler =
new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 21e44a3..5173b23 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -422,6 +422,21 @@ public class MockNodeManager implements NodeManager {
}
/**
+ * Remove the node stats and update the storage stats
+ * in this Node Manager.
+ *
+ * @param dnUuid UUID of the datanode.
+ */
+ @Override
+ public void processDeadNode(UUID dnUuid) {
+ SCMNodeStat stat = this.nodeMetricMap.get(dnUuid);
+ if (stat != null) {
+ aggregateStat.subtract(stat);
+ stat.set(0, 0, 0);
+ }
+ }
+
+ /**
* A class to declare some values for the nodes so that our tests
* won't fail.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
index 5ca9cb7..698332e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java
@@ -27,19 +27,27 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
import org.apache.hadoop.hdds.scm.container.Mapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import static org.mockito.Matchers.eq;
@@ -51,6 +59,29 @@ import org.mockito.Mockito;
public class TestDeadNodeHandler {
private List<ReplicationRequest> sentEvents = new ArrayList<>();
+ private SCMNodeManager nodeManager;
+ private Node2ContainerMap node2ContainerMap;
+ private ContainerStateManager containerStateManager;
+ private NodeReportHandler nodeReportHandler;
+ private DeadNodeHandler deadNodeHandler;
+ private EventPublisher publisher;
+ private EventQueue eventQueue;
+
+ @Before
+ public void setup() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ node2ContainerMap = new Node2ContainerMap();
+ containerStateManager = new ContainerStateManager(conf,
+ Mockito.mock(Mapping.class),
+ Mockito.mock(PipelineSelector.class));
+ eventQueue = new EventQueue();
+ nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue);
+ deadNodeHandler = new DeadNodeHandler(node2ContainerMap,
+ containerStateManager, nodeManager);
+ eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
+ publisher = Mockito.mock(EventPublisher.class);
+ nodeReportHandler = new NodeReportHandler(nodeManager);
+ }
@Test
public void testOnMessage() throws IOException {
@@ -58,13 +89,6 @@ public class TestDeadNodeHandler {
DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails();
DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails();
- Node2ContainerMap node2ContainerMap = new Node2ContainerMap();
- ContainerStateManager containerStateManager = new ContainerStateManager(
- new OzoneConfiguration(),
- Mockito.mock(Mapping.class),
- Mockito.mock(PipelineSelector.class)
- );
-
ContainerInfo container1 =
TestUtils.allocateContainer(containerStateManager);
ContainerInfo container2 =
@@ -72,9 +96,6 @@ public class TestDeadNodeHandler {
ContainerInfo container3 =
TestUtils.allocateContainer(containerStateManager);
- DeadNodeHandler handler =
- new DeadNodeHandler(node2ContainerMap, containerStateManager);
-
registerReplicas(node2ContainerMap, datanode1, container1, container2);
registerReplicas(node2ContainerMap, datanode2, container1, container3);
@@ -84,10 +105,8 @@ public class TestDeadNodeHandler {
TestUtils.closeContainer(containerStateManager, container1);
- EventPublisher publisher = Mockito.mock(EventPublisher.class);
-
//WHEN datanode1 is dead
- handler.onMessage(datanode1, publisher);
+ deadNodeHandler.onMessage(datanode1, publisher);
//THEN
//node2ContainerMap has not been changed
@@ -128,22 +147,76 @@ public class TestDeadNodeHandler {
replicationRequestParameter.getValue().getExpecReplicationCount());
}
- private void registerReplicas(ContainerStateManager containerStateManager,
+ @Test
+ public void testStatisticsUpdate() throws Exception {
+ //GIVEN
+ DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails();
+ DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails();
+ String storagePath1 = GenericTestUtils.getRandomizedTempPath()
+ .concat("/" + datanode1.getUuidString());
+ String storagePath2 = GenericTestUtils.getRandomizedTempPath()
+ .concat("/" + datanode2.getUuidString());
+
+ StorageReportProto storageOne = TestUtils.createStorageReport(
+ datanode1.getUuid(), storagePath1, 100, 10, 90, null);
+ StorageReportProto storageTwo = TestUtils.createStorageReport(
+ datanode2.getUuid(), storagePath2, 200, 20, 180, null);
+ nodeReportHandler.onMessage(getNodeReport(datanode1, storageOne),
+ Mockito.mock(EventPublisher.class));
+ nodeReportHandler.onMessage(getNodeReport(datanode2, storageTwo),
+ Mockito.mock(EventPublisher.class));
+
+ ContainerInfo container1 =
+ TestUtils.allocateContainer(containerStateManager);
+ registerReplicas(node2ContainerMap, datanode1, container1);
+
+ SCMNodeStat stat = nodeManager.getStats();
+ Assert.assertTrue(stat.getCapacity().get() == 300);
+ Assert.assertTrue(stat.getRemaining().get() == 270);
+ Assert.assertTrue(stat.getScmUsed().get() == 30);
+
+ SCMNodeMetric nodeStat = nodeManager.getNodeStat(datanode1);
+ Assert.assertTrue(nodeStat.get().getCapacity().get() == 100);
+ Assert.assertTrue(nodeStat.get().getRemaining().get() == 90);
+ Assert.assertTrue(nodeStat.get().getScmUsed().get() == 10);
+
+ //WHEN datanode1 is dead.
+ eventQueue.fireEvent(SCMEvents.DEAD_NODE, datanode1);
+ Thread.sleep(100);
+
+ //THEN statistics in SCM should changed.
+ stat = nodeManager.getStats();
+ Assert.assertTrue(stat.getCapacity().get() == 200);
+ Assert.assertTrue(stat.getRemaining().get() == 180);
+ Assert.assertTrue(stat.getScmUsed().get() == 20);
+
+ nodeStat = nodeManager.getNodeStat(datanode1);
+ Assert.assertTrue(nodeStat.get().getCapacity().get() == 0);
+ Assert.assertTrue(nodeStat.get().getRemaining().get() == 0);
+ Assert.assertTrue(nodeStat.get().getScmUsed().get() == 0);
+ }
+
+ private void registerReplicas(ContainerStateManager csm,
ContainerInfo container, DatanodeDetails... datanodes) {
- containerStateManager.getContainerStateMap()
+ csm.getContainerStateMap()
.addContainerReplica(new ContainerID(container.getContainerID()),
datanodes);
}
- private void registerReplicas(Node2ContainerMap node2ContainerMap,
+ private void registerReplicas(Node2ContainerMap node2ConMap,
DatanodeDetails datanode,
ContainerInfo... containers)
throws SCMException {
- node2ContainerMap
+ node2ConMap
.insertNewDatanode(datanode.getUuid(),
Arrays.stream(containers)
.map(container -> new ContainerID(container.getContainerID()))
.collect(Collectors.toSet()));
}
-}
\ No newline at end of file
+ private NodeReportFromDatanode getNodeReport(DatanodeDetails dn,
+ StorageReportProto... reports) {
+ NodeReportProto nodeReportProto = TestUtils.createNodeReport(reports);
+ return new NodeReportFromDatanode(dn, nodeReportProto);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/184544ef/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index e8a6892..4238703 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -307,4 +307,13 @@ public class ReplicationNodeManagerMock implements NodeManager {
EventPublisher publisher) {
// do nothing.
}
+
+ /**
+ * Empty implementation for processDeadNode.
+ * @param dnUuid
+ */
+ @Override
+ public void processDeadNode(UUID dnUuid) {
+ // do nothing.
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-8745. Misplaced the
TestRMWebServicesFairScheduler.java file. Contributed by Y. SREENIVASULU
REDDY.
Posted by in...@apache.org.
YARN-8745. Misplaced the TestRMWebServicesFairScheduler.java file. Contributed by Y. SREENIVASULU REDDY.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44edcdfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44edcdfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44edcdfd
Branch: refs/heads/HDFS-13532
Commit: 44edcdfd6ad09d2b1375c87706ba7c12e527a574
Parents: 94b5c54
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Sep 26 10:09:11 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Sep 26 10:09:11 2018 +0900
----------------------------------------------------------------------
.../webapp/TestRMWebServicesFairScheduler.java | 160 -------------------
...ustomResourceTypesConfigurationProvider.java | 4 +-
.../FairSchedulerJsonVerifications.java | 4 +-
.../FairSchedulerXmlVerifications.java | 4 +-
.../TestRMWebServicesFairScheduler.java | 160 +++++++++++++++++++
...ervicesFairSchedulerCustomResourceTypes.java | 4 +-
.../webapp/helper/AppInfoJsonVerifications.java | 4 +-
.../webapp/helper/AppInfoXmlVerifications.java | 4 +-
.../webapp/helper/BufferedClientResponse.java | 2 +-
.../ResourceRequestsJsonVerifications.java | 4 +-
.../ResourceRequestsXmlVerifications.java | 4 +-
11 files changed, 185 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
deleted file mode 100644
index 58c72ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
-
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
-import org.apache.hadoop.http.JettyUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
-
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
-import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Before;
-import org.junit.Test;
-import javax.ws.rs.core.MediaType;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Tests RM Webservices fair scheduler resources.
- */
-public class TestRMWebServicesFairScheduler extends JerseyTestBase {
- private static MockRM rm;
- private static YarnConfiguration conf;
-
- private static class WebServletModule extends ServletModule {
- @Override
- protected void configureServlets() {
- bind(JAXBContextResolver.class);
- bind(RMWebServices.class);
- bind(GenericExceptionHandler.class);
- conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
- ResourceScheduler.class);
- rm = new MockRM(conf);
- bind(ResourceManager.class).toInstance(rm);
- serve("/*").with(GuiceContainer.class);
- }
- }
-
- static {
- GuiceServletConfig
- .setInjector(Guice.createInjector(new WebServletModule()));
- }
-
- @Before
- @Override
- public void setUp() throws Exception {
- super.setUp();
- GuiceServletConfig
- .setInjector(Guice.createInjector(new WebServletModule()));
- }
-
- public TestRMWebServicesFairScheduler() {
- super(new WebAppDescriptor.Builder(
- "org.apache.hadoop.yarn.server.resourcemanager.webapp")
- .contextListenerClass(GuiceServletConfig.class)
- .filterClass(com.google.inject.servlet.GuiceFilter.class)
- .contextPath("jersey-guice-filter").servletPath("/").build());
- }
-
- @Test
- public void testClusterScheduler() throws JSONException {
- WebResource r = resource();
- ClientResponse response =
- r.path("ws").path("v1").path("cluster").path("scheduler")
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- JSONObject json = response.getEntity(JSONObject.class);
- verifyClusterScheduler(json);
- }
-
- @Test
- public void testClusterSchedulerSlash() throws JSONException {
- WebResource r = resource();
- ClientResponse response =
- r.path("ws").path("v1").path("cluster").path("scheduler/")
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- JSONObject json = response.getEntity(JSONObject.class);
- verifyClusterScheduler(json);
- }
-
- @Test
- public void testClusterSchedulerWithSubQueues()
- throws JSONException {
- FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
- QueueManager queueManager = scheduler.getQueueManager();
- // create LeafQueue
- queueManager.getLeafQueue("root.q.subqueue1", true);
- queueManager.getLeafQueue("root.q.subqueue2", true);
-
- WebResource r = resource();
- ClientResponse response =
- r.path("ws").path("v1").path("cluster").path("scheduler")
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- JSONObject json = response.getEntity(JSONObject.class);
- JSONArray subQueueInfo = json.getJSONObject("scheduler")
- .getJSONObject("schedulerInfo").getJSONObject("rootQueue")
- .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
- .getJSONObject("childQueues").getJSONArray("queue");
- // subQueueInfo is consist of subqueue1 and subqueue2 info
- assertEquals(2, subQueueInfo.length());
-
- // Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
- try {
- subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
- fail("FairSchedulerQueueInfo should omit field 'childQueues'"
- + "if child queue is empty.");
- } catch (JSONException je) {
- assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
- }
- }
-
- private void verifyClusterScheduler(JSONObject json) throws JSONException {
- assertEquals("incorrect number of elements", 1, json.length());
- JSONObject info = json.getJSONObject("scheduler");
- assertEquals("incorrect number of elements", 1, info.length());
- info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements", 2, info.length());
- JSONObject rootQueue = info.getJSONObject("rootQueue");
- assertEquals("root", rootQueue.getString("queueName"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
index bb1fce0..e80adc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
index 924411a..4d09660 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
index 63ae7b7..5ca2301 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java
new file mode 100644
index 0000000..58c72ee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
+
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import javax.ws.rs.core.MediaType;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests RM Webservices fair scheduler resources.
+ */
+public class TestRMWebServicesFairScheduler extends JerseyTestBase {
+ private static MockRM rm;
+ private static YarnConfiguration conf;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+ ResourceScheduler.class);
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ }
+ }
+
+ static {
+ GuiceServletConfig
+ .setInjector(Guice.createInjector(new WebServletModule()));
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ GuiceServletConfig
+ .setInjector(Guice.createInjector(new WebServletModule()));
+ }
+
+ public TestRMWebServicesFairScheduler() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testClusterScheduler() throws JSONException {
+ WebResource r = resource();
+ ClientResponse response =
+ r.path("ws").path("v1").path("cluster").path("scheduler")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject json = response.getEntity(JSONObject.class);
+ verifyClusterScheduler(json);
+ }
+
+ @Test
+ public void testClusterSchedulerSlash() throws JSONException {
+ WebResource r = resource();
+ ClientResponse response =
+ r.path("ws").path("v1").path("cluster").path("scheduler/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject json = response.getEntity(JSONObject.class);
+ verifyClusterScheduler(json);
+ }
+
+ @Test
+ public void testClusterSchedulerWithSubQueues()
+ throws JSONException {
+ FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+ QueueManager queueManager = scheduler.getQueueManager();
+ // create LeafQueue
+ queueManager.getLeafQueue("root.q.subqueue1", true);
+ queueManager.getLeafQueue("root.q.subqueue2", true);
+
+ WebResource r = resource();
+ ClientResponse response =
+ r.path("ws").path("v1").path("cluster").path("scheduler")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject json = response.getEntity(JSONObject.class);
+ JSONArray subQueueInfo = json.getJSONObject("scheduler")
+ .getJSONObject("schedulerInfo").getJSONObject("rootQueue")
+ .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
+ .getJSONObject("childQueues").getJSONArray("queue");
+ // subQueueInfo is consist of subqueue1 and subqueue2 info
+ assertEquals(2, subQueueInfo.length());
+
+ // Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
+ try {
+ subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
+ fail("FairSchedulerQueueInfo should omit field 'childQueues'"
+ + "if child queue is empty.");
+ } catch (JSONException je) {
+ assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
+ }
+ }
+
+ private void verifyClusterScheduler(JSONObject json) throws JSONException {
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("scheduler");
+ assertEquals("incorrect number of elements", 1, info.length());
+ info = info.getJSONObject("schedulerInfo");
+ assertEquals("incorrect number of elements", 2, info.length());
+ JSONObject rootQueue = info.getJSONObject("rootQueue");
+ assertEquals("root", rootQueue.getString("queueName"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
index de4d5a1..aa5f9a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
index 4ab1443..dea65fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
index 7c5b6db..97ffdfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
index a8990ca..92bccbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
index 6e58a89..19f6ca7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44edcdfd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
index af9b0f3..2611a07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-8774. Memory leak when
CapacityScheduler allocates from reserved container with non-default label.
Contributed by Tao Yang.
Posted by in...@apache.org.
YARN-8774. Memory leak when CapacityScheduler allocates from reserved container with non-default label. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8598b498
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8598b498
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8598b498
Branch: refs/heads/HDFS-13532
Commit: 8598b498bcaf4deffa822f871a26635bdf3d9d5c
Parents: 7093afd
Author: Eric E Payne <er...@oath.com>
Authored: Fri Sep 28 15:32:07 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Fri Sep 28 15:32:07 2018 +0000
----------------------------------------------------------------------
.../allocator/RegularContainerAllocator.java | 18 +++++-
.../TestNodeLabelContainerAllocation.java | 62 ++++++++++++++++++++
2 files changed, 79 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8598b498/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index c0a11a0..c692fdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -750,9 +750,25 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// When reserving container
RMContainer updatedContainer = reservedContainer;
if (updatedContainer == null) {
+ AppPlacementAllocator<FiCaSchedulerNode> ps =
+ application.getAppSchedulingInfo()
+ .getAppPlacementAllocator(schedulerKey);
+ if (null == ps) {
+ LOG.warn("Failed to get " + AppPlacementAllocator.class.getName()
+ + " for application=" + application.getApplicationId()
+ + " schedulerRequestKey=" + schedulerKey);
+ ActivitiesLogger.APP
+ .recordAppActivityWithoutAllocation(activitiesManager, node,
+ application, schedulerKey.getPriority(),
+ ActivityDiagnosticConstant.
+ PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST,
+ ActivityState.REJECTED);
+ return ContainerAllocation.PRIORITY_SKIPPED;
+ }
updatedContainer = new RMContainerImpl(container, schedulerKey,
application.getApplicationAttemptId(), node.getNodeID(),
- application.getAppSchedulingInfo().getUser(), rmContext);
+ application.getAppSchedulingInfo().getUser(), rmContext,
+ ps.getPrimaryRequestedNodePartition());
}
allocationResult.updatedContainer = updatedContainer;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8598b498/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 1836919..9cfddd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -547,6 +547,68 @@ public class TestNodeLabelContainerAllocation {
rm1.close();
}
+ @Test (timeout = 120000)
+ public void testRMContainerLeakInLeafQueue() throws Exception {
+ // set node -> label
+ mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x"));
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"),
+ NodeId.newInstance("h2", 0), toSet("x")));
+
+ // inject node label manager
+ MockRM rm1 =
+ new MockRM(TestUtils.getConfigurationWithDefaultQueueLabels(conf)) {
+ @Override public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // label = x
+ rm1.registerNode("h2:1234", 8 * GB); // label = x
+
+ // launch an app to queue a1 (label = x), and check all container will
+ // be allocated in h1
+ RMApp app1 = rm1.submitApp(1 * GB, "app1", "user", null, "a1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ RMApp app2 = rm1.submitApp(1 * GB, "app2", "user", null, "a1");
+ MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ // request a container.
+ am1.allocate("*", 7 * GB, 2, new ArrayList<ContainerId>());
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1");
+
+ // Do node heartbeats 1 time
+ // scheduler will reserve a container for app1
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ // Check if a 4G container allocated for app1, and 4G is reserved
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+ Assert.assertEquals(1, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(1, schedulerApp1.getReservedContainers().size());
+
+ // kill app2 then do node heartbeat 1 time
+ // scheduler will allocate a container from the reserved container on nm1
+ rm1.killApp(app2.getApplicationId());
+ rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED);
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Assert.assertEquals(2, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(0, schedulerApp1.getReservedContainers().size());
+
+ // After kill app1, LeafQueue#ignorePartitionExclusivityRMContainers should
+ // be clean, otherwise resource leak happened
+ rm1.killApp(app1.getApplicationId());
+ rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
+ Assert.assertEquals(0, leafQueue.getIgnoreExclusivityRMContainers().size());
+
+ rm1.close();
+ }
+
private void checkPendingResource(MockRM rm, int priority,
ApplicationAttemptId attemptId, int memory) {
CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-13768. Adding replicas to
volume map makes DataNode start slowly. Contributed by Surendra Singh
Lilhore.
Posted by in...@apache.org.
HDFS-13768. Adding replicas to volume map makes DataNode start slowly. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56893557
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56893557
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56893557
Branch: refs/heads/HDFS-13532
Commit: 5689355783de005ebc604f4403dc5129a286bfca
Parents: f6c5ef9
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Oct 2 09:43:14 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Oct 2 09:43:14 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../hadoop/hdfs/server/datanode/DataNode.java | 2 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 177 +++++++++++++++++--
.../datanode/fsdataset/impl/FsDatasetImpl.java | 14 ++
.../datanode/fsdataset/impl/FsDatasetUtil.java | 30 ++--
.../datanode/fsdataset/impl/FsVolumeList.java | 5 +-
.../datanode/fsdataset/impl/ReplicaMap.java | 25 +++
.../src/main/resources/hdfs-default.xml | 9 +
.../fsdataset/impl/TestFsVolumeList.java | 64 ++++++-
9 files changed, 300 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index a7e7b9b..d8024dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -365,6 +365,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT = 500;
public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated";
public static final int DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT = 0;
+ public static final String
+ DFS_DATANODE_VOLUMES_REPLICA_ADD_THREADPOOL_SIZE_KEY =
+ "dfs.datanode.volumes.replica-add.threadpool.size";
public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose";
public static final boolean DFS_DATANODE_SYNCONCLOSE_DEFAULT = false;
public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index c980395..270e30b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1695,7 +1695,7 @@ public class DataNode extends ReconfigurableBase
return blockPoolManager.get(bpid);
}
- int getBpOsCount() {
+ public int getBpOsCount() {
return blockPoolManager.getAllNamenodeThreads().size();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 2adfb6b..b9b581f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -28,8 +28,19 @@ import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.RandomAccessFile;
import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
import java.util.Scanner;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinTask;
+import java.util.concurrent.RecursiveAction;
import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
@@ -52,8 +63,8 @@ import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBuilder;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaTracker.RamDiskReplica;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
-
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.DiskChecker;
@@ -96,6 +107,17 @@ class BlockPoolSlice {
private final int maxDataLength;
private final FileIoProvider fileIoProvider;
+ private static ForkJoinPool addReplicaThreadPool = null;
+ private static final int VOLUMES_REPLICA_ADD_THREADPOOL_SIZE = Runtime
+ .getRuntime().availableProcessors();
+ private static final Comparator<File> FILE_COMPARATOR =
+ new Comparator<File>() {
+ @Override
+ public int compare(File f1, File f2) {
+ return f1.getName().compareTo(f2.getName());
+ }
+ };
+
// TODO:FEDERATION scalability issue - a thread per DU is needed
private final GetSpaceUsed dfsUsage;
@@ -161,13 +183,15 @@ class BlockPoolSlice {
.setConf(conf)
.setInitialUsed(loadDfsUsed())
.build();
-
+ // initialize add replica fork join pool
+ initializeAddReplicaPool(conf);
// Make the dfs usage to be saved during shutdown.
shutdownHook = new Runnable() {
@Override
public void run() {
if (!dfsUsedSaved) {
saveDfsUsed();
+ addReplicaThreadPool.shutdownNow();
}
}
};
@@ -175,6 +199,21 @@ class BlockPoolSlice {
SHUTDOWN_HOOK_PRIORITY);
}
+ private synchronized void initializeAddReplicaPool(Configuration conf) {
+ if (addReplicaThreadPool == null) {
+ FsDatasetImpl dataset = (FsDatasetImpl) volume.getDataset();
+ int numberOfBlockPoolSlice = dataset.getVolumeCount()
+ * dataset.getBPServiceCount();
+ int poolsize = Math.max(numberOfBlockPoolSlice,
+ VOLUMES_REPLICA_ADD_THREADPOOL_SIZE);
+ // Default pool sizes is max of (volume * number of bp_service) and
+ // number of processor.
+ addReplicaThreadPool = new ForkJoinPool(conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_VOLUMES_REPLICA_ADD_THREADPOOL_SIZE_KEY,
+ poolsize));
+ }
+ }
+
File getDirectory() {
return currentDir.getParentFile();
}
@@ -374,10 +413,55 @@ class BlockPoolSlice {
boolean success = readReplicasFromCache(volumeMap, lazyWriteReplicaMap);
if (!success) {
+ List<IOException> exceptions = Collections
+ .synchronizedList(new ArrayList<IOException>());
+ Queue<RecursiveAction> subTaskQueue =
+ new ConcurrentLinkedQueue<RecursiveAction>();
+
// add finalized replicas
- addToReplicasMap(volumeMap, finalizedDir, lazyWriteReplicaMap, true);
+ AddReplicaProcessor task = new AddReplicaProcessor(volumeMap,
+ finalizedDir, lazyWriteReplicaMap, true, exceptions, subTaskQueue);
+ ForkJoinTask<Void> finalizedTask = addReplicaThreadPool.submit(task);
+
// add rbw replicas
- addToReplicasMap(volumeMap, rbwDir, lazyWriteReplicaMap, false);
+ task = new AddReplicaProcessor(volumeMap, rbwDir, lazyWriteReplicaMap,
+ false, exceptions, subTaskQueue);
+ ForkJoinTask<Void> rbwTask = addReplicaThreadPool.submit(task);
+
+ try {
+ finalizedTask.get();
+ rbwTask.get();
+ } catch (InterruptedException | ExecutionException e) {
+ exceptions.add(new IOException(
+ "Failed to start sub tasks to add replica in replica map :"
+ + e.getMessage()));
+ }
+
+ //wait for all the tasks to finish.
+ waitForSubTaskToFinish(subTaskQueue, exceptions);
+ }
+ }
+
+ /**
+ * Wait till all the recursive task for add replica to volume completed.
+ *
+ * @param subTaskQueue
+ * {@link AddReplicaProcessor} tasks list.
+ * @param exceptions
+ * exceptions occurred in sub tasks.
+ * @throws IOException
+ * throw if any sub task or multiple sub tasks failed.
+ */
+ private void waitForSubTaskToFinish(Queue<RecursiveAction> subTaskQueue,
+ List<IOException> exceptions) throws IOException {
+ while (!subTaskQueue.isEmpty()) {
+ RecursiveAction task = subTaskQueue.poll();
+ if (task != null) {
+ task.join();
+ }
+ }
+ if (!exceptions.isEmpty()) {
+ throw MultipleIOException.createIOException(exceptions);
}
}
@@ -526,10 +610,10 @@ class BlockPoolSlice {
}
}
- ReplicaInfo oldReplica = volumeMap.get(bpid, newReplica.getBlockId());
- if (oldReplica == null) {
- volumeMap.add(bpid, newReplica);
- } else {
+ ReplicaInfo tmpReplicaInfo = volumeMap.addAndGet(bpid, newReplica);
+ ReplicaInfo oldReplica = (tmpReplicaInfo == newReplica) ? null
+ : tmpReplicaInfo;
+ if (oldReplica != null) {
// We have multiple replicas of the same block so decide which one
// to keep.
newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
@@ -558,15 +642,23 @@ class BlockPoolSlice {
* storage.
* @param isFinalized true if the directory has finalized replicas;
* false if the directory has rbw replicas
+ * @param exceptions list of exception which need to return to parent thread.
+ * @param subTaskQueue queue of sub tasks
*/
void addToReplicasMap(ReplicaMap volumeMap, File dir,
- final RamDiskReplicaTracker lazyWriteReplicaMap,
- boolean isFinalized)
+ final RamDiskReplicaTracker lazyWriteReplicaMap, boolean isFinalized,
+ List<IOException> exceptions, Queue<RecursiveAction> subTaskQueue)
throws IOException {
File[] files = fileIoProvider.listFiles(volume, dir);
- for (File file : files) {
+ Arrays.sort(files, FILE_COMPARATOR);
+ for (int i = 0; i < files.length; i++) {
+ File file = files[i];
if (file.isDirectory()) {
- addToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
+ // Launch new sub task.
+ AddReplicaProcessor subTask = new AddReplicaProcessor(volumeMap, file,
+ lazyWriteReplicaMap, isFinalized, exceptions, subTaskQueue);
+ subTask.fork();
+ subTaskQueue.add(subTask);
}
if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) {
@@ -581,7 +673,7 @@ class BlockPoolSlice {
}
long genStamp = FsDatasetUtil.getGenerationStampFromFile(
- files, file);
+ files, file, i);
long blockId = Block.filename2id(file.getName());
Block block = new Block(blockId, file.length(), genStamp);
addReplicaToReplicasMap(block, volumeMap, lazyWriteReplicaMap,
@@ -886,4 +978,63 @@ class BlockPoolSlice {
public long getNumOfBlocks() {
return numOfBlocks.get();
}
+
+ /**
+ * Recursive action for add replica in map.
+ */
+ class AddReplicaProcessor extends RecursiveAction {
+
+ private ReplicaMap volumeMap;
+ private File dir;
+ private RamDiskReplicaTracker lazyWriteReplicaMap;
+ private boolean isFinalized;
+ private List<IOException> exceptions;
+ private Queue<RecursiveAction> subTaskQueue;
+
+ /**
+ * @param volumeMap
+ * the replicas map
+ * @param dir
+ * an input directory
+ * @param lazyWriteReplicaMap
+ * Map of replicas on transient storage.
+ * @param isFinalized
+ * true if the directory has finalized replicas; false if the
+ * directory has rbw replicas
+ * @param exceptions
+ * List of exception which need to return to parent thread.
+ * @param subTaskQueue
+ * queue of sub tasks
+ */
+ AddReplicaProcessor(ReplicaMap volumeMap, File dir,
+ RamDiskReplicaTracker lazyWriteReplicaMap, boolean isFinalized,
+ List<IOException> exceptions, Queue<RecursiveAction> subTaskQueue) {
+ this.volumeMap = volumeMap;
+ this.dir = dir;
+ this.lazyWriteReplicaMap = lazyWriteReplicaMap;
+ this.isFinalized = isFinalized;
+ this.exceptions = exceptions;
+ this.subTaskQueue = subTaskQueue;
+ }
+
+ @Override
+ protected void compute() {
+ try {
+ addToReplicasMap(volumeMap, dir, lazyWriteReplicaMap, isFinalized,
+ exceptions, subTaskQueue);
+ } catch (IOException e) {
+ LOG.warn("Caught exception while adding replicas from " + volume
+ + " in subtask. Will throw later.", e);
+ exceptions.add(e);
+ }
+ }
+ }
+
+ /**
+ * Return the size of fork pool used for adding replica in map.
+ */
+ @VisibleForTesting
+ public static int getAddReplicaForkPoolSize() {
+ return addReplicaThreadPool.getPoolSize();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 27196c2..027a0bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -3307,6 +3307,20 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
this.timer = newTimer;
}
+ /**
+ * Return the number of BP service count.
+ */
+ public int getBPServiceCount() {
+ return datanode.getBpOsCount();
+ }
+
+ /**
+ * Return the number of volume.
+ */
+ public int getVolumeCount() {
+ return volumes.getVolumes().size();
+ }
+
void stopAllDataxceiverThreads(FsVolumeImpl volume) {
try (AutoCloseableLock lock = datasetLock.acquire()) {
for (String blockPoolId : volumeMap.getBlockPoolList()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
index 9f115a0..8a3b237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
@@ -117,21 +117,27 @@ public class FsDatasetUtil {
}
/**
- * Find the meta-file for the specified block file
- * and then return the generation stamp from the name of the meta-file.
+ * Find the meta-file for the specified block file and then return the
+ * generation stamp from the name of the meta-file. Generally meta file will
+ * be the next file in sorted array of file's.
+ *
+ * @param listdir
+ * sorted list of file based on name.
+ * @param blockFile
+ * block file for which generation stamp is needed.
+ * @param index
+ * index of block file in array.
+ * @return generation stamp for block file.
*/
- static long getGenerationStampFromFile(File[] listdir, File blockFile)
- throws IOException {
+ static long getGenerationStampFromFile(File[] listdir, File blockFile,
+ int index) {
String blockName = blockFile.getName();
- for (int j = 0; j < listdir.length; j++) {
- String path = listdir[j].getName();
- if (!path.startsWith(blockName)) {
- continue;
- }
- if (blockFile.getCanonicalPath().equals(listdir[j].getCanonicalPath())) {
- continue;
+ if ((index + 1) < listdir.length) {
+ // Check if next index file is meta file
+ String metaFile = listdir[index + 1].getName();
+ if (metaFile.startsWith(blockName)) {
+ return Block.getGenerationStamp(metaFile);
}
- return Block.getGenerationStamp(listdir[j].getName());
}
FsDatasetImpl.LOG.warn("Block " + blockFile + " does not have a metafile!");
return HdfsConstants.GRANDFATHER_GENERATION_STAMP;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index a1804ae..a0fcb54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -226,8 +226,9 @@ class FsVolumeList {
throw exceptions.get(0);
}
long totalTimeTaken = Time.monotonicNow() - totalStartTime;
- FsDatasetImpl.LOG.info("Total time to add all replicas to map: "
- + totalTimeTaken + "ms");
+ FsDatasetImpl.LOG
+ .info("Total time to add all replicas to map for block pool " + bpid
+ + ": " + totalTimeTaken + "ms");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
index d4fb69b..c630b95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
@@ -136,6 +136,31 @@ class ReplicaMap {
}
/**
+ * Add a replica's meta information into the map, if already exist
+ * return the old replicaInfo.
+ */
+ ReplicaInfo addAndGet(String bpid, ReplicaInfo replicaInfo) {
+ checkBlockPool(bpid);
+ checkBlock(replicaInfo);
+ try (AutoCloseableLock l = lock.acquire()) {
+ FoldedTreeSet<ReplicaInfo> set = map.get(bpid);
+ if (set == null) {
+ // Add an entry for block pool if it does not exist already
+ set = new FoldedTreeSet<>();
+ map.put(bpid, set);
+ }
+ ReplicaInfo oldReplicaInfo = set.get(replicaInfo.getBlockId(),
+ LONG_AND_BLOCK_COMPARATOR);
+ if (oldReplicaInfo != null) {
+ return oldReplicaInfo;
+ } else {
+ set.add(replicaInfo);
+ }
+ return replicaInfo;
+ }
+ }
+
+ /**
* Add all entries from the given replica map into the local replica map.
*/
void addAll(ReplicaMap other) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 1573582..2ee8399 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1282,6 +1282,15 @@
</property>
<property>
+ <name>dfs.datanode.volumes.replica-add.threadpool.size</name>
+ <value></value>
+ <description>Specifies the maximum number of threads to use for
+ adding block in volume. Default value for this configuration is
+ max of (volume * number of bp_service, number of processor).
+ </description>
+</property>
+
+<property>
<name>dfs.image.compress</name>
<value>false</value>
<description>When this value is true, the dfs image will be compressed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56893557/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index f53c21c..581a7a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -18,11 +18,16 @@
package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
import com.google.common.base.Supplier;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DF;
import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
@@ -30,6 +35,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -40,12 +46,16 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -315,4 +325,56 @@ public class TestFsVolumeList {
.build();
assertEquals(600, volume4.getReserved());
}
-}
+
+ @Test(timeout = 60000)
+ public void testAddRplicaProcessorForAddingReplicaInMap() throws Exception {
+ Configuration cnf = new Configuration();
+ int poolSize = 5;
+ cnf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ cnf.setInt(
+ DFSConfigKeys.DFS_DATANODE_VOLUMES_REPLICA_ADD_THREADPOOL_SIZE_KEY,
+ poolSize);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(cnf).numDataNodes(1)
+ .storagesPerDatanode(1).build();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ // Generate data blocks.
+ ExecutorService pool = Executors.newFixedThreadPool(10);
+ List<Future<?>> futureList = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ for (int j = 0; j < 10; j++) {
+ try {
+ DFSTestUtil.createFile(fs, new Path("File_" + getName() + j), 10,
+ (short) 1, 0);
+ } catch (IllegalArgumentException | IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ };
+ thread.setName("FileWriter" + i);
+ futureList.add(pool.submit(thread));
+ }
+ // Wait for data generation
+ for (Future<?> f : futureList) {
+ f.get();
+ }
+ fs.close();
+ FsDatasetImpl fsDataset = (FsDatasetImpl) cluster.getDataNodes().get(0)
+ .getFSDataset();
+ ReplicaMap volumeMap = new ReplicaMap(new AutoCloseableLock());
+ RamDiskReplicaTracker ramDiskReplicaMap = RamDiskReplicaTracker
+ .getInstance(conf, fsDataset);
+ FsVolumeImpl vol = (FsVolumeImpl) fsDataset.getFsVolumeReferences().get(0);
+ String bpid = cluster.getNamesystem().getBlockPoolId();
+ // It will create BlockPoolSlice.AddReplicaProcessor task's and lunch in
+ // ForkJoinPool recursively
+ vol.getVolumeMap(bpid, volumeMap, ramDiskReplicaMap);
+ assertTrue("Failed to add all the replica to map", volumeMap.replicas(bpid)
+ .size() == 1000);
+ assertTrue("Fork pool size should be " + poolSize,
+ BlockPoolSlice.getAddReplicaForkPoolSize() == poolSize);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-8829. Cluster metrics can
fail with IndexOutOfBoundsException. Contributed by Akshay Agarwal.
Posted by in...@apache.org.
YARN-8829. Cluster metrics can fail with IndexOutOfBoundsException. Contributed by Akshay Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7093afd8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7093afd8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7093afd8
Branch: refs/heads/HDFS-13532
Commit: 7093afd8746cc120875eb27053bad692482d2aab
Parents: 284aca8
Author: bibinchundatt <bi...@apache.org>
Authored: Fri Sep 28 12:32:37 2018 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Fri Sep 28 12:35:33 2018 +0530
----------------------------------------------------------------------
.../server/router/clientrm/FederationClientInterceptor.java | 2 +-
.../router/clientrm/TestFederationClientInterceptor.java | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7093afd8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index 3d9b394..797b1aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -654,7 +654,7 @@ public class FederationClientInterceptor
exceptions.put(subClusterId, ioe);
}
}
- if (results.isEmpty()) {
+ if (results.isEmpty() && !clusterIds.isEmpty()) {
SubClusterId subClusterId = clusterIds.get(0);
IOException ioe = exceptions.get(subClusterId);
if (ioe != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7093afd8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java
index 8f19ecc..c49fe4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
@@ -412,5 +413,13 @@ public class TestFederationClientInterceptor extends BaseRouterClientRMTest {
interceptor.getClusterMetrics(GetClusterMetricsRequest.newInstance());
Assert.assertEquals(subClusters.size(),
response.getClusterMetrics().getNumNodeManagers());
+
+ ClientMethod remoteMethod = new ClientMethod("getClusterMetrics",
+ new Class[] {GetClusterMetricsRequest.class},
+ new Object[] {GetClusterMetricsRequest.newInstance()});
+ Map<SubClusterId, GetClusterMetricsResponse> clusterMetrics =interceptor.
+ invokeConcurrent(new ArrayList<>(), remoteMethod,
+ GetClusterMetricsResponse.class);
+ Assert.assertEquals(true, clusterMetrics.isEmpty());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-8665. Added Yarn service
cancel upgrade option. Contributed by Chandni Singh
Posted by in...@apache.org.
YARN-8665. Added Yarn service cancel upgrade option.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/913f87da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/913f87da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/913f87da
Branch: refs/heads/HDFS-13532
Commit: 913f87dada27776c539dfb352400ecf8d40e7943
Parents: e0ff8e2
Author: Eric Yang <ey...@apache.org>
Authored: Wed Sep 26 14:51:35 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Sep 26 14:51:35 2018 -0400
----------------------------------------------------------------------
.../yarn/service/client/ApiServiceClient.java | 20 ++
.../hadoop/yarn/service/webapp/ApiServer.java | 34 +-
.../hadoop/yarn/service/ClientAMProtocol.java | 5 +
.../hadoop/yarn/service/ClientAMService.java | 12 +
.../hadoop/yarn/service/ServiceEvent.java | 14 +-
.../hadoop/yarn/service/ServiceEventType.java | 3 +-
.../hadoop/yarn/service/ServiceManager.java | 331 +++++++++++++------
.../service/api/records/ContainerState.java | 2 +-
.../yarn/service/api/records/ServiceState.java | 7 +-
.../yarn/service/client/ServiceClient.java | 21 ++
.../yarn/service/component/Component.java | 275 ++++++++++-----
.../yarn/service/component/ComponentEvent.java | 10 -
.../service/component/ComponentEventType.java | 1 +
.../yarn/service/component/ComponentState.java | 3 +-
.../component/instance/ComponentInstance.java | 269 ++++++++++++---
.../instance/ComponentInstanceEventType.java | 3 +-
.../instance/ComponentInstanceState.java | 3 +-
.../containerlaunch/ContainerLaunchService.java | 4 +-
.../pb/client/ClientAMProtocolPBClientImpl.java | 13 +
.../service/ClientAMProtocolPBServiceImpl.java | 13 +
.../yarn/service/provider/ProviderUtils.java | 9 +-
.../yarn/service/utils/ServiceApiUtil.java | 14 +-
.../yarn/service/utils/SliderFileSystem.java | 49 +++
.../src/main/proto/ClientAMProtocol.proto | 8 +
.../yarn/service/MockRunningServiceContext.java | 20 +-
.../hadoop/yarn/service/ServiceTestUtils.java | 2 +-
.../hadoop/yarn/service/TestServiceManager.java | 136 ++++++--
.../yarn/service/TestYarnNativeServices.java | 44 +++
.../yarn/service/client/TestServiceCLI.java | 17 +
.../yarn/service/component/TestComponent.java | 239 +++++++++++--
.../instance/TestComponentInstance.java | 177 +++++++++-
.../src/test/resources/log4j.properties | 19 ++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 12 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +
.../hadoop/yarn/client/api/AppAdminClient.java | 13 +
.../containermanager/ContainerManagerImpl.java | 1 +
36 files changed, 1470 insertions(+), 335 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index ca6cc50..b7a1541 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -723,4 +723,24 @@ public class ApiServiceClient extends AppAdminClient {
}
return null;
}
+
+ @Override
+ public int actionCancelUpgrade(
+ String appName) throws IOException, YarnException {
+ int result;
+ try {
+ Service service = new Service();
+ service.setName(appName);
+ service.setState(ServiceState.CANCEL_UPGRADING);
+ String buffer = jsonSerDeser.toJson(service);
+ LOG.info("Cancel upgrade in progress. Please wait..");
+ ClientResponse response = getApiClient(getServicePath(appName))
+ .put(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Failed to cancel upgrade: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index cd6f0d7..c4e3317 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -58,6 +58,7 @@ import java.util.*;
import java.util.stream.Collectors;
import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
+import static org.apache.hadoop.yarn.service.api.records.ServiceState.CANCEL_UPGRADING;
import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
@@ -445,6 +446,12 @@ public class ApiServer {
return upgradeService(updateServiceData, ugi);
}
+ // If CANCEL_UPGRADING is requested
+ if (updateServiceData.getState() != null &&
+ updateServiceData.getState() == CANCEL_UPGRADING) {
+ return cancelUpgradeService(appName, ugi);
+ }
+
// If new lifetime value specified then update it
if (updateServiceData.getLifetime() != null
&& updateServiceData.getLifetime() > 0) {
@@ -460,8 +467,7 @@ public class ApiServer {
LOG.error(message, e);
return formatResponse(Status.NOT_FOUND, e.getMessage());
} catch (YarnException e) {
- String message = "Service is not found in hdfs: " + appName;
- LOG.error(message, e);
+ LOG.error(e.getMessage(), e);
return formatResponse(Status.NOT_FOUND, e.getMessage());
} catch (Exception e) {
String message = "Error while performing operation for app: " + appName;
@@ -707,6 +713,27 @@ public class ApiServer {
return formatResponse(Status.ACCEPTED, status);
}
+ private Response cancelUpgradeService(String serviceName,
+ final UserGroupInformation ugi) throws IOException, InterruptedException {
+ int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ int exitCode = sc.actionCancelUpgrade(serviceName);
+ sc.close();
+ return exitCode;
+ });
+ if (result == EXIT_SUCCESS) {
+ ServiceStatus status = new ServiceStatus();
+ LOG.info("Service {} cancelling upgrade", serviceName);
+ status.setDiagnostics("Service " + serviceName +
+ " cancelling upgrade.");
+ status.setState(ServiceState.ACCEPTED);
+ return formatResponse(Status.ACCEPTED, status);
+ }
+ return Response.status(Status.BAD_REQUEST).build();
+ }
+
private Response processComponentsUpgrade(UserGroupInformation ugi,
String serviceName, Set<String> compNames) throws YarnException,
IOException, InterruptedException {
@@ -734,7 +761,8 @@ public class ApiServer {
Service service, List<Container> containers) throws YarnException,
IOException, InterruptedException {
- if (service.getState() != ServiceState.UPGRADING) {
+ if (!service.getState().equals(ServiceState.UPGRADING) &&
+ !service.getState().equals(ServiceState.UPGRADING_AUTO_FINALIZE)) {
throw new YarnException(
String.format("The upgrade of service %s has not been initiated.",
service.getName()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
index 652a314..39e7dfa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.service;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
@@ -60,4 +62,7 @@ public interface ClientAMProtocol {
GetCompInstancesResponseProto getCompInstances(
GetCompInstancesRequestProto request) throws IOException, YarnException;
+
+ CancelUpgradeResponseProto cancelUpgrade(
+ CancelUpgradeRequestProto request) throws IOException, YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index 2ef8f7e..47e9829 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
@@ -208,4 +210,14 @@ public class ClientAMService extends AbstractService
ServiceApiUtil.CONTAINER_JSON_SERDE.toJson(containers.toArray(
new Container[containers.size()]))).build();
}
+
+ @Override
+ public CancelUpgradeResponseProto cancelUpgrade(
+ CancelUpgradeRequestProto request) throws IOException, YarnException {
+ LOG.info("Cancel service upgrade by {}",
+ UserGroupInformation.getCurrentUser());
+ ServiceEvent event = new ServiceEvent(ServiceEventType.CANCEL_UPGRADE);
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
+ return CancelUpgradeResponseProto.newBuilder().build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java
index 3a55472..cf4455d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.service;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.hadoop.yarn.service.api.records.Component;
-import java.util.Queue;
+import java.util.List;
/**
* Events are handled by {@link ServiceManager} to manage the service
@@ -33,7 +33,8 @@ public class ServiceEvent extends AbstractEvent<ServiceEventType> {
private String version;
private boolean autoFinalize;
private boolean expressUpgrade;
- private Queue<Component> compsToUpgradeInOrder;
+ // For express upgrade they should be in order.
+ private List<Component> compsToUpgrade;
public ServiceEvent(ServiceEventType serviceEventType) {
super(serviceEventType);
@@ -71,13 +72,12 @@ public class ServiceEvent extends AbstractEvent<ServiceEventType> {
return this;
}
- public Queue<Component> getCompsToUpgradeInOrder() {
- return compsToUpgradeInOrder;
+ public List<Component> getCompsToUpgrade() {
+ return compsToUpgrade;
}
- public ServiceEvent setCompsToUpgradeInOrder(
- Queue<Component> compsToUpgradeInOrder) {
- this.compsToUpgradeInOrder = compsToUpgradeInOrder;
+ public ServiceEvent setCompsToUpgrade(List<Component> compsToUpgrade) {
+ this.compsToUpgrade = compsToUpgrade;
return this;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java
index 4fc420b..03afdd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java
@@ -24,5 +24,6 @@ package org.apache.hadoop.yarn.service;
public enum ServiceEventType {
START,
UPGRADE,
- CHECK_STABLE
+ CHECK_STABLE,
+ CANCEL_UPGRADE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
index 04454b1..4851325 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
@@ -20,32 +20,31 @@ package org.apache.hadoop.yarn.service;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.service.api.records.ComponentState;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.service.component.ComponentEvent;
import org.apache.hadoop.yarn.service.component.ComponentEventType;
import org.apache.hadoop.yarn.service.component.ComponentRestartPolicy;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.MultipleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.state.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.MessageFormat;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
@@ -71,8 +70,10 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
private final SliderFileSystem fs;
private String upgradeVersion;
- private Queue<org.apache.hadoop.yarn.service.api.records
- .Component> compsToUpgradeInOrder;
+ private List<org.apache.hadoop.yarn.service.api.records
+ .Component> componentsToUpgrade;
+ private List<String> compsAffectedByUpgrade = new ArrayList<>();
+ private String cancelledVersion;
private static final StateMachineFactory<ServiceManager, State,
ServiceEventType, ServiceEvent> STATE_MACHINE_FACTORY =
@@ -88,11 +89,14 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
.addTransition(State.UPGRADING, EnumSet.of(State.STABLE,
State.UPGRADING), ServiceEventType.START,
- new CheckStableTransition())
+ new StartFromUpgradeTransition())
.addTransition(State.UPGRADING, EnumSet.of(State.STABLE,
State.UPGRADING), ServiceEventType.CHECK_STABLE,
new CheckStableTransition())
+
+ .addTransition(State.UPGRADING, State.UPGRADING,
+ ServiceEventType.CANCEL_UPGRADE, new CancelUpgradeTransition())
.installTopology();
public ServiceManager(ServiceContext context) {
@@ -148,19 +152,25 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
public State transition(ServiceManager serviceManager,
ServiceEvent event) {
serviceManager.upgradeVersion = event.getVersion();
+ serviceManager.componentsToUpgrade = event.getCompsToUpgrade();
+ event.getCompsToUpgrade().forEach(comp ->
+ serviceManager.compsAffectedByUpgrade.add(comp.getName()));
try {
if (event.isExpressUpgrade()) {
- serviceManager.serviceSpec.setState(ServiceState.EXPRESS_UPGRADING);
- serviceManager.compsToUpgradeInOrder = event
- .getCompsToUpgradeInOrder();
- serviceManager.upgradeNextCompIfAny();
+ serviceManager.dispatchNeedUpgradeEvents(false);
+ serviceManager.upgradeNextCompIfAny(false);
+ } else {
+ serviceManager.dispatchNeedUpgradeEvents(false);
+ }
+
+ if (event.isExpressUpgrade()) {
+ serviceManager.setServiceState(ServiceState.EXPRESS_UPGRADING);
} else if (event.isAutoFinalize()) {
- serviceManager.serviceSpec.setState(ServiceState
- .UPGRADING_AUTO_FINALIZE);
+ serviceManager.setServiceState(ServiceState.UPGRADING_AUTO_FINALIZE);
} else {
- serviceManager.serviceSpec.setState(
- ServiceState.UPGRADING);
+ serviceManager.setServiceState(ServiceState.UPGRADING);
}
+
return State.UPGRADING;
} catch (Throwable e) {
LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(),
@@ -181,24 +191,32 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
if (currState.equals(ServiceState.STABLE)) {
return State.STABLE;
}
- if (currState.equals(ServiceState.EXPRESS_UPGRADING)) {
- org.apache.hadoop.yarn.service.api.records.Component component =
- serviceManager.compsToUpgradeInOrder.peek();
- if (!component.getState().equals(ComponentState.NEEDS_UPGRADE) &&
- !component.getState().equals(ComponentState.UPGRADING)) {
- serviceManager.compsToUpgradeInOrder.remove();
+ if (currState.equals(ServiceState.EXPRESS_UPGRADING) ||
+ currState.equals(ServiceState.CANCEL_UPGRADING)) {
+
+ if (!serviceManager.componentsToUpgrade.isEmpty()) {
+ org.apache.hadoop.yarn.service.api.records.Component compSpec =
+ serviceManager.componentsToUpgrade.get(0);
+ Component component = serviceManager.scheduler.getAllComponents()
+ .get(compSpec.getName());
+
+ if (!component.isUpgrading()) {
+ serviceManager.componentsToUpgrade.remove(0);
+ serviceManager.upgradeNextCompIfAny(
+ currState.equals(ServiceState.CANCEL_UPGRADING));
+ }
}
- serviceManager.upgradeNextCompIfAny();
}
+
if (currState.equals(ServiceState.UPGRADING_AUTO_FINALIZE) ||
- event.getType().equals(ServiceEventType.START) ||
- (currState.equals(ServiceState.EXPRESS_UPGRADING) &&
- serviceManager.compsToUpgradeInOrder.isEmpty())) {
+ ((currState.equals(ServiceState.EXPRESS_UPGRADING) ||
+ currState.equals(ServiceState.CANCEL_UPGRADING)) &&
+ serviceManager.componentsToUpgrade.isEmpty())) {
+
ServiceState targetState = checkIfStable(serviceManager.serviceSpec);
if (targetState.equals(ServiceState.STABLE)) {
- if (serviceManager.finalizeUpgrade()) {
- LOG.info("Service def state changed from {} -> {}", currState,
- serviceManager.serviceSpec.getState());
+ if (serviceManager.finalizeUpgrade(
+ currState.equals(ServiceState.CANCEL_UPGRADING))) {
return State.STABLE;
}
}
@@ -207,52 +225,149 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
}
}
- private void upgradeNextCompIfAny() {
- if (!compsToUpgradeInOrder.isEmpty()) {
+ private static class StartFromUpgradeTransition implements
+ MultipleArcTransition<ServiceManager, ServiceEvent, State> {
+
+ @Override
+ public State transition(ServiceManager serviceManager, ServiceEvent event) {
+ ServiceState currState = serviceManager.serviceSpec.getState();
+
+ ServiceState targetState = checkIfStable(serviceManager.serviceSpec);
+ if (targetState.equals(ServiceState.STABLE)) {
+ if (serviceManager.finalizeUpgrade(
+ currState.equals(ServiceState.CANCEL_UPGRADING))) {
+ return State.STABLE;
+ }
+ }
+ return State.UPGRADING;
+ }
+ }
+
+ private static class CancelUpgradeTransition implements
+ SingleArcTransition<ServiceManager, ServiceEvent> {
+
+ @Override
+ public void transition(ServiceManager serviceManager,
+ ServiceEvent event) {
+ if (!serviceManager.getState().equals(State.UPGRADING)) {
+ LOG.info("[SERVICE]: Cannot cancel the upgrade in {} state",
+ serviceManager.getState());
+ return;
+ }
+ try {
+ Service targetSpec = ServiceApiUtil.loadService(
+ serviceManager.context.fs, serviceManager.getName());
+
+ Service sourceSpec = ServiceApiUtil.loadServiceUpgrade(
+ serviceManager.context.fs, serviceManager.getName(),
+ serviceManager.upgradeVersion);
+ serviceManager.cancelledVersion = serviceManager.upgradeVersion;
+ LOG.info("[SERVICE] cancel version {}",
+ serviceManager.cancelledVersion);
+ serviceManager.upgradeVersion = serviceManager.serviceSpec.getVersion();
+ serviceManager.componentsToUpgrade = serviceManager
+ .resolveCompsToUpgrade(sourceSpec, targetSpec);
+
+ serviceManager.compsAffectedByUpgrade.clear();
+ serviceManager.componentsToUpgrade.forEach(comp ->
+ serviceManager.compsAffectedByUpgrade.add(comp.getName()));
+
+ serviceManager.dispatchNeedUpgradeEvents(true);
+ serviceManager.upgradeNextCompIfAny(true);
+ serviceManager.setServiceState(ServiceState.CANCEL_UPGRADING);
+ } catch (Throwable e) {
+ LOG.error("[SERVICE]: Cancellation of upgrade failed", e);
+ }
+ }
+ }
+
+ private void upgradeNextCompIfAny(boolean cancelUpgrade) {
+ if (!componentsToUpgrade.isEmpty()) {
org.apache.hadoop.yarn.service.api.records.Component component =
- compsToUpgradeInOrder.peek();
+ componentsToUpgrade.get(0);
+
+ serviceSpec.getComponent(component.getName()).getContainers().forEach(
+ container -> {
+ ComponentInstanceEvent upgradeEvent = new ComponentInstanceEvent(
+ ContainerId.fromString(container.getId()),
+ !cancelUpgrade ? ComponentInstanceEventType.UPGRADE :
+ ComponentInstanceEventType.CANCEL_UPGRADE);
+ LOG.info("Upgrade container {} {}", container.getId(),
+ cancelUpgrade);
+ dispatcher.getEventHandler().handle(upgradeEvent);
+ });
+ }
+ }
- ComponentEvent needUpgradeEvent = new ComponentEvent(
- component.getName(), ComponentEventType.UPGRADE).setTargetSpec(
- component).setUpgradeVersion(upgradeVersion).setExpressUpgrade(true);
- context.scheduler.getDispatcher().getEventHandler().handle(
- needUpgradeEvent);
+ private void dispatchNeedUpgradeEvents(boolean cancelUpgrade) {
+ if (componentsToUpgrade != null) {
+ componentsToUpgrade.forEach(component -> {
+ ComponentEvent needUpgradeEvent = new ComponentEvent(
+ component.getName(), !cancelUpgrade ? ComponentEventType.UPGRADE :
+ ComponentEventType.CANCEL_UPGRADE)
+ .setTargetSpec(component)
+ .setUpgradeVersion(upgradeVersion);
+ LOG.info("Upgrade component {} {}", component.getName(), cancelUpgrade);
+ context.scheduler.getDispatcher().getEventHandler()
+ .handle(needUpgradeEvent);
+ });
}
}
/**
* @return whether finalization of upgrade was successful.
*/
- private boolean finalizeUpgrade() {
- try {
- // save the application id and state to
- Service targetSpec = ServiceApiUtil.loadServiceUpgrade(
- fs, getName(), upgradeVersion);
- targetSpec.setId(serviceSpec.getId());
- targetSpec.setState(ServiceState.STABLE);
- Map<String, Component> allComps = scheduler.getAllComponents();
- targetSpec.getComponents().forEach(compSpec -> {
- Component comp = allComps.get(compSpec.getName());
- compSpec.setState(comp.getComponentSpec().getState());
- });
- jsonSerDeser.save(fs.getFileSystem(),
- ServiceApiUtil.getServiceJsonPath(fs, getName()), targetSpec, true);
- fs.deleteClusterUpgradeDir(getName(), upgradeVersion);
- } catch (IOException e) {
- LOG.error("Upgrade did not complete because unable to re-write the" +
- " service definition", e);
- return false;
+ private boolean finalizeUpgrade(boolean cancelUpgrade) {
+ if (!cancelUpgrade) {
+ try {
+ // save the application id and state to
+ Service targetSpec = ServiceApiUtil.loadServiceUpgrade(
+ fs, getName(), upgradeVersion);
+ targetSpec.setId(serviceSpec.getId());
+ targetSpec.setState(ServiceState.STABLE);
+ Map<String, Component> allComps = scheduler.getAllComponents();
+ targetSpec.getComponents().forEach(compSpec -> {
+ Component comp = allComps.get(compSpec.getName());
+ compSpec.setState(comp.getComponentSpec().getState());
+ });
+ jsonSerDeser.save(fs.getFileSystem(),
+ ServiceApiUtil.getServiceJsonPath(fs, getName()), targetSpec, true);
+ } catch (IOException e) {
+ LOG.error("Upgrade did not complete because unable to re-write the" +
+ " service definition", e);
+ return false;
+ }
}
try {
- fs.deleteClusterUpgradeDir(getName(), upgradeVersion);
+ String upgradeVersionToDel = cancelUpgrade? cancelledVersion :
+ upgradeVersion;
+ LOG.info("[SERVICE]: delete upgrade dir version {}", upgradeVersionToDel);
+ fs.deleteClusterUpgradeDir(getName(), upgradeVersionToDel);
+
+ for (String comp : compsAffectedByUpgrade) {
+ String compDirVersionToDel = cancelUpgrade? cancelledVersion :
+ serviceSpec.getVersion();
+ LOG.info("[SERVICE]: delete {} dir version {}", comp,
+ compDirVersionToDel);
+ fs.deleteComponentDir(compDirVersionToDel, comp);
+ }
+
+ if (cancelUpgrade) {
+ fs.deleteComponentsVersionDirIfEmpty(cancelledVersion);
+ } else {
+ fs.deleteComponentsVersionDirIfEmpty(serviceSpec.getVersion());
+ }
+
} catch (IOException e) {
LOG.warn("Unable to delete upgrade definition for service {} " +
"version {}", getName(), upgradeVersion);
}
- serviceSpec.setState(ServiceState.STABLE);
+ setServiceState(ServiceState.STABLE);
serviceSpec.setVersion(upgradeVersion);
upgradeVersion = null;
+ cancelledVersion = null;
+ compsAffectedByUpgrade.clear();
return true;
}
@@ -291,30 +406,8 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
context.fs, context.service.getName(), upgradeVersion);
List<org.apache.hadoop.yarn.service.api.records.Component>
- compsNeedUpgradeList = componentsFinder.
- findTargetComponentSpecs(context.service, targetSpec);
-
- // remove all components from need upgrade list if there restart policy
- // doesn't all upgrade.
- if (compsNeedUpgradeList != null) {
- compsNeedUpgradeList.removeIf(component -> {
- org.apache.hadoop.yarn.service.api.records.Component.RestartPolicyEnum
- restartPolicy = component.getRestartPolicy();
-
- final ComponentRestartPolicy restartPolicyHandler =
- Component.getRestartPolicyHandler(restartPolicy);
- // Do not allow upgrades for components which have NEVER/ON_FAILURE
- // restart policy
- if (!restartPolicyHandler.allowUpgrades()) {
- LOG.info("The component {} has a restart policy that doesnt " +
- "allow upgrades {} ", component.getName(),
- component.getRestartPolicy().toString());
- return true;
- }
-
- return false;
- });
- }
+ compsNeedUpgradeList = resolveCompsToUpgrade(context.service,
+ targetSpec);
ServiceEvent event = new ServiceEvent(ServiceEventType.UPGRADE)
.setVersion(upgradeVersion)
@@ -334,7 +427,7 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
List<String> resolvedComps = ServiceApiUtil
.resolveCompsDependency(targetSpec);
- Queue<org.apache.hadoop.yarn.service.api.records.Component>
+ List<org.apache.hadoop.yarn.service.api.records.Component>
orderedCompUpgrade = new LinkedList<>();
resolvedComps.forEach(compName -> {
org.apache.hadoop.yarn.service.api.records.Component component =
@@ -343,30 +436,68 @@ public class ServiceManager implements EventHandler<ServiceEvent> {
orderedCompUpgrade.add(component);
}
});
- event.setCompsToUpgradeInOrder(orderedCompUpgrade);
+ event.setCompsToUpgrade(orderedCompUpgrade);
+ } else {
+ event.setCompsToUpgrade(compsNeedUpgradeList);
}
+ context.scheduler.getDispatcher().getEventHandler().handle(
+ event);
- context.scheduler.getDispatcher().getEventHandler().handle(event);
-
- if (compsNeedUpgradeList != null && !compsNeedUpgradeList.isEmpty()) {
- if (!expressUpgrade) {
- compsNeedUpgradeList.forEach(component -> {
- ComponentEvent needUpgradeEvent = new ComponentEvent(
- component.getName(), ComponentEventType.UPGRADE).setTargetSpec(
- component).setUpgradeVersion(event.getVersion());
- context.scheduler.getDispatcher().getEventHandler().handle(
- needUpgradeEvent);
-
- });
- }
- } else if (autoFinalize) {
- // nothing to upgrade if upgrade auto finalize is requested, trigger a
+ if (autoFinalize && (compsNeedUpgradeList == null ||
+ compsNeedUpgradeList.isEmpty())) {
+ // nothing to upgrade and auto finalize is requested, trigger a
// state check.
context.scheduler.getDispatcher().getEventHandler().handle(
new ServiceEvent(ServiceEventType.CHECK_STABLE));
}
}
+ private List<org.apache.hadoop.yarn.service.api.records.Component>
+ resolveCompsToUpgrade(Service sourceSpec, Service targetSpec) {
+
+ List<org.apache.hadoop.yarn.service.api.records.Component>
+ compsNeedUpgradeList = componentsFinder.
+ findTargetComponentSpecs(sourceSpec, targetSpec);
+
+ // remove all components from need upgrade list if there restart policy
+ // doesn't all upgrade.
+ if (compsNeedUpgradeList != null) {
+ compsNeedUpgradeList.removeIf(component -> {
+ org.apache.hadoop.yarn.service.api.records.Component.RestartPolicyEnum
+ restartPolicy = component.getRestartPolicy();
+
+ final ComponentRestartPolicy restartPolicyHandler =
+ Component.getRestartPolicyHandler(restartPolicy);
+ // Do not allow upgrades for components which have NEVER/ON_FAILURE
+ // restart policy
+ if (!restartPolicyHandler.allowUpgrades()) {
+ LOG.info("The component {} has a restart policy that doesnt " +
+ "allow upgrades {} ", component.getName(),
+ component.getRestartPolicy().toString());
+ return true;
+ }
+
+ return false;
+ });
+ }
+
+ return compsNeedUpgradeList;
+ }
+
+ /**
+ * Sets the state of the service in the service spec.
+ * @param state service state
+ */
+ private void setServiceState(
+ org.apache.hadoop.yarn.service.api.records.ServiceState state) {
+ org.apache.hadoop.yarn.service.api.records.ServiceState curState =
+ serviceSpec.getState();
+ if (!curState.equals(state)) {
+ serviceSpec.setState(state);
+ LOG.info("[SERVICE] spec state changed from {} -> {}", curState, state);
+ }
+ }
+
/**
* Returns the name of the service.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
index cac527a..a6e9a2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ContainerState.java
@@ -27,5 +27,5 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
public enum ContainerState {
RUNNING_BUT_UNREADY, READY, STOPPED, NEEDS_UPGRADE, UPGRADING, SUCCEEDED,
- FAILED;
+ FAILED, FAILED_UPGRADE;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
index 49c1985..3f2f4f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
@@ -29,5 +29,10 @@ import org.apache.hadoop.classification.InterfaceStability;
@ApiModel(description = "The current state of an service.")
public enum ServiceState {
ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX, UPGRADING,
- UPGRADING_AUTO_FINALIZE, EXPRESS_UPGRADING, SUCCEEDED;
+ UPGRADING_AUTO_FINALIZE, EXPRESS_UPGRADING, SUCCEEDED, CANCEL_UPGRADING;
+
+ public static boolean isUpgrading(ServiceState state) {
+ return state.equals(UPGRADING) || state.equals(UPGRADING_AUTO_FINALIZE)
+ || state.equals(EXPRESS_UPGRADING);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index a27ed87..23db57e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.CancelUpgradeRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.CompInstancesUpgradeRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
@@ -353,6 +354,26 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
}
@Override
+ public int actionCancelUpgrade(String appName) throws IOException,
+ YarnException {
+ Service liveService = getStatus(appName);
+ if (liveService == null ||
+ !ServiceState.isUpgrading(liveService.getState())) {
+ throw new YarnException("Service " + appName + " is not upgrading, " +
+ "so nothing to cancel.");
+ }
+
+ ApplicationReport appReport = yarnClient.getApplicationReport(
+ getAppId(appName));
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ throw new YarnException(appName + " AM hostname is empty");
+ }
+ ClientAMProtocol proxy = createAMProxy(appName, appReport);
+ proxy.cancelUpgrade(CancelUpgradeRequestProto.newBuilder().build());
+ return EXIT_SUCCESS;
+ }
+
+ @Override
public int actionCleanUp(String appName, String userName) throws
IOException, YarnException {
if (cleanUpRegistry(appName, userName)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index acf3404..526bde0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.service.component;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import static org.apache.hadoop.yarn.service.api.records.Component
@@ -44,7 +43,6 @@ import org.apache.hadoop.yarn.service.ServiceEventType;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.api.records.ResourceInformation;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
import org.apache.hadoop.yarn.service.ContainerFailureTracker;
import org.apache.hadoop.yarn.service.ServiceContext;
@@ -89,6 +87,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.CANCEL_UPGRADE;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.UPGRADE;
import static org.apache.hadoop.yarn.service.component.ComponentState.*;
import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
@@ -126,9 +126,8 @@ public class Component implements EventHandler<ComponentEvent> {
new ConcurrentHashMap<>();
private boolean healthThresholdMonitorEnabled = false;
- private AtomicBoolean upgradeInProgress = new AtomicBoolean(false);
- private ComponentEvent upgradeEvent;
- private AtomicLong numContainersThatNeedUpgrade = new AtomicLong(0);
+ private UpgradeStatus upgradeStatus = new UpgradeStatus();
+ private UpgradeStatus cancelUpgradeStatus = new UpgradeStatus();
private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
stateMachine;
@@ -160,6 +159,8 @@ public class Component implements EventHandler<ComponentEvent> {
// Flex while previous flex is still in progress
.addTransition(FLEXING, EnumSet.of(FLEXING, STABLE), FLEX,
new FlexComponentTransition())
+ .addTransition(FLEXING, EnumSet.of(UPGRADING, FLEXING, STABLE),
+ CHECK_STABLE, new CheckStableTransition())
// container failed while stable
.addTransition(STABLE, FLEXING, CONTAINER_COMPLETED,
@@ -172,19 +173,28 @@ public class Component implements EventHandler<ComponentEvent> {
// For flex down, go to STABLE state
.addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
FLEX, new FlexComponentTransition())
- .addTransition(STABLE, UPGRADING, ComponentEventType.UPGRADE,
- new ComponentNeedsUpgradeTransition())
- //Upgrade while previous upgrade is still in progress
- .addTransition(UPGRADING, UPGRADING, ComponentEventType.UPGRADE,
- new ComponentNeedsUpgradeTransition())
- .addTransition(UPGRADING, EnumSet.of(UPGRADING, FLEXING, STABLE),
- CHECK_STABLE, new CheckStableTransition())
- .addTransition(FLEXING, EnumSet.of(UPGRADING, FLEXING, STABLE),
- CHECK_STABLE, new CheckStableTransition())
+ .addTransition(STABLE, UPGRADING, UPGRADE,
+ new NeedsUpgradeTransition())
+ .addTransition(STABLE, CANCEL_UPGRADING, CANCEL_UPGRADE,
+ new NeedsUpgradeTransition())
.addTransition(STABLE, EnumSet.of(STABLE), CHECK_STABLE,
new CheckStableTransition())
- .addTransition(UPGRADING, FLEXING, CONTAINER_COMPLETED,
- new ContainerCompletedTransition())
+
+ // Cancel upgrade while previous upgrade is still in progress
+ .addTransition(UPGRADING, CANCEL_UPGRADING,
+ CANCEL_UPGRADE, new NeedsUpgradeTransition())
+ .addTransition(UPGRADING, EnumSet.of(UPGRADING, STABLE),
+ CHECK_STABLE, new CheckStableTransition())
+ .addTransition(UPGRADING, UPGRADING, CONTAINER_COMPLETED,
+ new CompletedAfterUpgradeTransition())
+
+ .addTransition(CANCEL_UPGRADING, EnumSet.of(CANCEL_UPGRADING, FLEXING,
+ STABLE), CHECK_STABLE, new CheckStableTransition())
+ .addTransition(CANCEL_UPGRADING, CANCEL_UPGRADING,
+ CONTAINER_COMPLETED, new CompletedAfterUpgradeTransition())
+ .addTransition(CANCEL_UPGRADING, FLEXING, CONTAINER_ALLOCATED,
+ new ContainerAllocatedTransition())
+
.installTopology();
public Component(
@@ -332,7 +342,7 @@ public class Component implements EventHandler<ComponentEvent> {
+ before + " to " + event.getDesired());
component.requestContainers(delta);
component.createNumCompInstances(delta);
- component.componentSpec.setState(
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState.FLEXING);
component.getScheduler().getApp().setState(ServiceState.STARTED);
return FLEXING;
@@ -430,11 +440,11 @@ public class Component implements EventHandler<ComponentEvent> {
if (component.getNumRunningInstances() + component
.getNumSucceededInstances() + component.getNumFailedInstances()
< component.getComponentSpec().getNumberOfContainers()) {
- component.componentSpec.setState(
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState.FLEXING);
return FLEXING;
} else{
- component.componentSpec.setState(
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState.STABLE);
return STABLE;
}
@@ -444,22 +454,22 @@ public class Component implements EventHandler<ComponentEvent> {
Component component) {
// if desired == running
if (component.componentMetrics.containersReady.value() == component
- .getComponentSpec().getNumberOfContainers()
- && component.numContainersThatNeedUpgrade.get() == 0) {
- component.componentSpec.setState(
+ .getComponentSpec().getNumberOfContainers() &&
+ !component.doesNeedUpgrade()) {
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState.STABLE);
return STABLE;
+ } else if (component.doesNeedUpgrade()) {
+ component.setComponentState(org.apache.hadoop.yarn.service.api.records.
+ ComponentState.NEEDS_UPGRADE);
+ return component.getState();
} else if (component.componentMetrics.containersReady.value() != component
.getComponentSpec().getNumberOfContainers()) {
- component.componentSpec.setState(
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState.FLEXING);
return FLEXING;
- } else {
- // component.numContainersThatNeedUpgrade.get() > 0
- component.componentSpec.setState(org.apache.hadoop.yarn.service.api.
- records.ComponentState.NEEDS_UPGRADE);
- return UPGRADING;
}
+ return component.getState();
}
// This method should be called whenever there is an increment or decrement
@@ -467,22 +477,16 @@ public class Component implements EventHandler<ComponentEvent> {
//This should not matter for terminating components
private static synchronized void checkAndUpdateComponentState(
Component component, boolean isIncrement) {
- org.apache.hadoop.yarn.service.api.records.ComponentState curState =
- component.componentSpec.getState();
if (component.getRestartPolicyHandler().isLongLived()) {
if (isIncrement) {
// check if all containers are in READY state
- if (component.numContainersThatNeedUpgrade.get() == 0
- && component.componentMetrics.containersReady.value()
- == component.componentMetrics.containersDesired.value()) {
- component.componentSpec.setState(
+ if (!component.upgradeStatus.areContainersUpgrading() &&
+ !component.cancelUpgradeStatus.areContainersUpgrading() &&
+ component.componentMetrics.containersReady.value() ==
+ component.componentMetrics.containersDesired.value()) {
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState.STABLE);
- if (curState != component.componentSpec.getState()) {
- LOG.info("[COMPONENT {}] state changed from {} -> {}",
- component.componentSpec.getName(), curState,
- component.componentSpec.getState());
- }
// component state change will trigger re-check of service state
component.context.getServiceManager().checkAndUpdateServiceState();
}
@@ -491,19 +495,14 @@ public class Component implements EventHandler<ComponentEvent> {
// still need to verify the count before changing the component state
if (component.componentMetrics.containersReady.value()
< component.componentMetrics.containersDesired.value()) {
- component.componentSpec.setState(
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState
.FLEXING);
} else if (component.componentMetrics.containersReady.value()
== component.componentMetrics.containersDesired.value()) {
- component.componentSpec.setState(
+ component.setComponentState(
org.apache.hadoop.yarn.service.api.records.ComponentState.STABLE);
}
- if (curState != component.componentSpec.getState()) {
- LOG.info("[COMPONENT {}] state changed from {} -> {}",
- component.componentSpec.getName(), curState,
- component.componentSpec.getState());
- }
// component state change will trigger re-check of service state
component.context.getServiceManager().checkAndUpdateServiceState();
}
@@ -511,8 +510,8 @@ public class Component implements EventHandler<ComponentEvent> {
// component state change will trigger re-check of service state
component.context.getServiceManager().checkAndUpdateServiceState();
}
- // when the service is stable then the state of component needs to
- // transition to stable
+ // triggers the state machine in component to reach appropriate state
+ // once the state in spec is changed.
component.dispatcher.getEventHandler().handle(
new ComponentEvent(component.getName(),
ComponentEventType.CHECK_STABLE));
@@ -544,25 +543,43 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
- private static class ComponentNeedsUpgradeTransition extends BaseTransition {
+ private static class CompletedAfterUpgradeTransition extends BaseTransition {
@Override
public void transition(Component component, ComponentEvent event) {
- component.upgradeInProgress.set(true);
- component.upgradeEvent = event;
- component.componentSpec.setState(org.apache.hadoop.yarn.service.api.
- records.ComponentState.NEEDS_UPGRADE);
- component.numContainersThatNeedUpgrade.set(
+ Preconditions.checkNotNull(event.getContainerId());
+ component.updateMetrics(event.getStatus());
+ component.dispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getContainerId(), STOP)
+ .setStatus(event.getStatus()));
+ }
+ }
+
+ private static class NeedsUpgradeTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ boolean isCancel = event.getType().equals(CANCEL_UPGRADE);
+ UpgradeStatus status = !isCancel ? component.upgradeStatus :
+ component.cancelUpgradeStatus;
+
+ status.inProgress.set(true);
+ status.targetSpec = event.getTargetSpec();
+ status.targetVersion = event.getUpgradeVersion();
+ LOG.info("[COMPONENT {}]: need upgrade to {}",
+ component.getName(), status.targetVersion);
+
+ status.containersNeedUpgrade.set(
component.componentSpec.getNumberOfContainers());
- component.componentSpec.getContainers().forEach(container -> {
- container.setState(ContainerState.NEEDS_UPGRADE);
- if (event.isExpressUpgrade()) {
- ComponentInstanceEvent upgradeEvent = new ComponentInstanceEvent(
- ContainerId.fromString(container.getId()),
- ComponentInstanceEventType.UPGRADE);
- LOG.info("Upgrade container {}", container.getId());
- component.dispatcher.getEventHandler().handle(upgradeEvent);
- }
+
+ component.setComponentState(org.apache.hadoop.yarn.service.api.
+ records.ComponentState.NEEDS_UPGRADE);
+
+ component.getAllComponentInstances().forEach(instance -> {
+ instance.setContainerState(ContainerState.NEEDS_UPGRADE);
});
+
+ if (event.getType().equals(CANCEL_UPGRADE)) {
+ component.upgradeStatus.reset();
+ }
}
}
@@ -572,22 +589,22 @@ public class Component implements EventHandler<ComponentEvent> {
@Override
public ComponentState transition(Component component,
ComponentEvent componentEvent) {
- org.apache.hadoop.yarn.service.api.records.ComponentState currState =
- component.componentSpec.getState();
- if (currState.equals(org.apache.hadoop.yarn.service.api.records
- .ComponentState.STABLE)) {
- return ComponentState.STABLE;
- }
// checkIfStable also updates the state in definition when STABLE
ComponentState targetState = checkIfStable(component);
- if (targetState.equals(STABLE) && component.upgradeInProgress.get()) {
- component.componentSpec.overwrite(
- component.upgradeEvent.getTargetSpec());
- component.upgradeEvent = null;
+
+ if (targetState.equals(STABLE) &&
+ !(component.upgradeStatus.isCompleted() &&
+ component.cancelUpgradeStatus.isCompleted())) {
+ // Component stable after upgrade or cancel upgrade
+ UpgradeStatus status = !component.cancelUpgradeStatus.isCompleted() ?
+ component.cancelUpgradeStatus : component.upgradeStatus;
+
+ component.componentSpec.overwrite(status.getTargetSpec());
+ status.reset();
+
ServiceEvent checkStable = new ServiceEvent(ServiceEventType.
CHECK_STABLE);
component.dispatcher.getEventHandler().handle(checkStable);
- component.upgradeInProgress.set(false);
}
return targetState;
}
@@ -625,11 +642,14 @@ public class Component implements EventHandler<ComponentEvent> {
"[COMPONENT {}]: Assigned {} to component instance {} and launch on host {} ",
getName(), container.getId(), instance.getCompInstanceName(),
container.getNodeId());
- if (upgradeInProgress.get()) {
+ if (!(upgradeStatus.isCompleted() && cancelUpgradeStatus.isCompleted())) {
+ UpgradeStatus status = !cancelUpgradeStatus.isCompleted() ?
+ cancelUpgradeStatus : upgradeStatus;
+
scheduler.getContainerLaunchService()
.launchCompInstance(scheduler.getApp(), instance, container,
- createLaunchContext(upgradeEvent.getTargetSpec(),
- upgradeEvent.getUpgradeVersion()));
+ createLaunchContext(status.getTargetSpec(),
+ status.getTargetVersion()));
} else {
scheduler.getContainerLaunchService().launchCompInstance(
scheduler.getApp(), instance, container,
@@ -830,6 +850,12 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
+ private boolean doesNeedUpgrade() {
+ return cancelUpgradeStatus.areContainersUpgrading() ||
+ upgradeStatus.areContainersUpgrading() ||
+ upgradeStatus.failed.get();
+ }
+
public boolean areDependenciesReady() {
List<String> dependencies = componentSpec.getDependencies();
if (ServiceUtils.isEmpty(dependencies)) {
@@ -911,10 +937,6 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
- public void decContainersThatNeedUpgrade() {
- numContainersThatNeedUpgrade.decrementAndGet();
- }
-
public int getNumReadyInstances() {
return componentMetrics.containersReady.value();
}
@@ -972,10 +994,33 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
- public ComponentEvent getUpgradeEvent() {
+ /**
+ * Returns whether a component is upgrading or not.
+ */
+ public boolean isUpgrading() {
+ this.readLock.lock();
+
+ try {
+ return !(upgradeStatus.isCompleted() &&
+ cancelUpgradeStatus.isCompleted());
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ public UpgradeStatus getUpgradeStatus() {
+ this.readLock.lock();
+ try {
+ return upgradeStatus;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ public UpgradeStatus getCancelUpgradeStatus() {
this.readLock.lock();
try {
- return upgradeEvent;
+ return cancelUpgradeStatus;
} finally {
this.readLock.unlock();
}
@@ -1013,6 +1058,70 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
+ /**
+ * Sets the state of the component in the component spec.
+ * @param state component state
+ */
+ private void setComponentState(
+ org.apache.hadoop.yarn.service.api.records.ComponentState state) {
+ org.apache.hadoop.yarn.service.api.records.ComponentState curState =
+ componentSpec.getState();
+ if (!curState.equals(state)) {
+ componentSpec.setState(state);
+ LOG.info("[COMPONENT {}] spec state changed from {} -> {}",
+ componentSpec.getName(), curState, state);
+ }
+ }
+
+ /**
+ * Status of upgrade.
+ */
+ public static class UpgradeStatus {
+ private org.apache.hadoop.yarn.service.api.records.Component targetSpec;
+ private String targetVersion;
+ private AtomicBoolean inProgress = new AtomicBoolean(false);
+ private AtomicLong containersNeedUpgrade = new AtomicLong(0);
+ private AtomicBoolean failed = new AtomicBoolean(false);
+
+ public org.apache.hadoop.yarn.service.api.records.
+ Component getTargetSpec() {
+ return targetSpec;
+ }
+
+ public String getTargetVersion() {
+ return targetVersion;
+ }
+
+ /*
+ * @return whether the upgrade is completed or not
+ */
+ public boolean isCompleted() {
+ return !inProgress.get();
+ }
+
+ public void decContainersThatNeedUpgrade() {
+ if (inProgress.get()) {
+ containersNeedUpgrade.decrementAndGet();
+ }
+ }
+
+ public void containerFailedUpgrade() {
+ failed.set(true);
+ }
+
+ void reset() {
+ containersNeedUpgrade.set(0);
+ targetSpec = null;
+ targetVersion = null;
+ inProgress.set(false);
+ failed.set(false);
+ }
+
+ boolean areContainersUpgrading() {
+ return containersNeedUpgrade.get() != 0;
+ }
+ }
+
public ServiceContext getContext() {
return context;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
index 643961d..84caa77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -35,7 +35,6 @@ public class ComponentEvent extends AbstractEvent<ComponentEventType> {
private ContainerId containerId;
private org.apache.hadoop.yarn.service.api.records.Component targetSpec;
private String upgradeVersion;
- private boolean expressUpgrade;
public ContainerId getContainerId() {
return containerId;
@@ -114,13 +113,4 @@ public class ComponentEvent extends AbstractEvent<ComponentEventType> {
this.upgradeVersion = upgradeVersion;
return this;
}
-
- public boolean isExpressUpgrade() {
- return expressUpgrade;
- }
-
- public ComponentEvent setExpressUpgrade(boolean expressUpgrade) {
- this.expressUpgrade = expressUpgrade;
- return this;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
index 44d781f..d211f49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
@@ -24,6 +24,7 @@ public enum ComponentEventType {
CONTAINER_RECOVERED,
CONTAINER_STARTED,
CONTAINER_COMPLETED,
+ CANCEL_UPGRADE,
UPGRADE,
CHECK_STABLE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
index 0f63d03..e1cd0c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
@@ -22,5 +22,6 @@ public enum ComponentState {
INIT,
FLEXING,
STABLE,
- UPGRADING
+ UPGRADING,
+ CANCEL_UPGRADING
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index afd8c67..89c9a22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
@@ -64,8 +65,10 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.Date;
+import java.util.EnumSet;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -84,8 +87,9 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
LoggerFactory.getLogger(ComponentInstance.class);
private static final String FAILED_BEFORE_LAUNCH_DIAG =
"failed before launch";
+ private static final String UPGRADE_FAILED = "upgrade failed";
- private StateMachine<ComponentInstanceState, ComponentInstanceEventType,
+ private StateMachine<ComponentInstanceState, ComponentInstanceEventType,
ComponentInstanceEvent> stateMachine;
private Component component;
private final ReadLock readLock;
@@ -106,7 +110,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
// This container object is used for rest API query
private org.apache.hadoop.yarn.service.api.records.Container containerSpec;
private String serviceVersion;
-
+ private AtomicBoolean upgradeInProgress = new AtomicBoolean(false);
+ private boolean pendingCancelUpgrade = false;
private static final StateMachineFactory<ComponentInstance,
ComponentInstanceState, ComponentInstanceEventType,
@@ -132,13 +137,23 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.addTransition(READY, STARTED, BECOME_NOT_READY,
new ContainerBecomeNotReadyTransition())
.addTransition(READY, INIT, STOP, new ContainerStoppedTransition())
- .addTransition(READY, UPGRADING, UPGRADE,
- new ContainerUpgradeTransition())
- .addTransition(UPGRADING, UPGRADING, UPGRADE,
- new ContainerUpgradeTransition())
- .addTransition(UPGRADING, READY, BECOME_READY,
- new ContainerBecomeReadyTransition())
- .addTransition(UPGRADING, INIT, STOP, new ContainerStoppedTransition())
+ .addTransition(READY, UPGRADING, UPGRADE, new UpgradeTransition())
+ .addTransition(READY, EnumSet.of(READY, CANCEL_UPGRADING), CANCEL_UPGRADE,
+ new CancelUpgradeTransition())
+
+ // FROM UPGRADING
+ .addTransition(UPGRADING, EnumSet.of(READY, CANCEL_UPGRADING),
+ CANCEL_UPGRADE, new CancelUpgradeTransition())
+ .addTransition(UPGRADING, EnumSet.of(READY), BECOME_READY,
+ new ReadyAfterUpgradeTransition())
+ .addTransition(UPGRADING, UPGRADING, STOP,
+ new StoppedAfterUpgradeTransition())
+
+ // FROM CANCEL_UPGRADING
+ .addTransition(CANCEL_UPGRADING, EnumSet.of(CANCEL_UPGRADING, READY),
+ BECOME_READY, new ReadyAfterUpgradeTransition())
+ .addTransition(CANCEL_UPGRADING, EnumSet.of(CANCEL_UPGRADING, INIT),
+ STOP, new StoppedAfterCancelUpgradeTransition())
.installTopology();
public ComponentInstance(Component component,
@@ -217,24 +232,53 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
@Override
public void transition(ComponentInstance compInstance,
ComponentInstanceEvent event) {
- compInstance.containerSpec.setState(ContainerState.READY);
- if (compInstance.getState().equals(ComponentInstanceState.UPGRADING)) {
- compInstance.component.incContainersReady(false);
- compInstance.component.decContainersThatNeedUpgrade();
- compInstance.serviceVersion = compInstance.component.getUpgradeEvent()
- .getUpgradeVersion();
- ComponentEvent checkState = new ComponentEvent(
- compInstance.component.getName(), ComponentEventType.CHECK_STABLE);
- compInstance.scheduler.getDispatcher().getEventHandler().handle(
- checkState);
+ compInstance.setContainerState(ContainerState.READY);
+ compInstance.component.incContainersReady(true);
+ compInstance.postContainerReady();
+ }
+ }
- } else {
- compInstance.component.incContainersReady(true);
- }
- if (compInstance.timelineServiceEnabled) {
- compInstance.serviceTimelinePublisher
- .componentInstanceBecomeReady(compInstance.containerSpec);
+ private static class ReadyAfterUpgradeTransition implements
+ MultipleArcTransition<ComponentInstance, ComponentInstanceEvent,
+ ComponentInstanceState> {
+
+ @Override
+ public ComponentInstanceState transition(ComponentInstance instance,
+ ComponentInstanceEvent event) {
+
+ if (instance.pendingCancelUpgrade) {
+ // cancellation of upgrade was triggered before the upgrade was
+ // finished.
+ LOG.info("{} received ready but cancellation pending",
+ event.getContainerId());
+ instance.upgradeInProgress.set(true);
+ instance.cancelUpgrade();
+ instance.pendingCancelUpgrade = false;
+ return instance.getState();
}
+
+ instance.upgradeInProgress.set(false);
+ instance.setContainerState(ContainerState.READY);
+ instance.component.incContainersReady(false);
+
+ Component.UpgradeStatus status = instance.getState().equals(UPGRADING) ?
+ instance.component.getUpgradeStatus() :
+ instance.component.getCancelUpgradeStatus();
+ status.decContainersThatNeedUpgrade();
+
+ instance.serviceVersion = status.getTargetVersion();
+ ComponentEvent checkState = new ComponentEvent(
+ instance.component.getName(),
+ ComponentEventType.CHECK_STABLE);
+ instance.scheduler.getDispatcher().getEventHandler().handle(checkState);
+ instance.postContainerReady();
+ return ComponentInstanceState.READY;
+ }
+ }
+
+ private void postContainerReady() {
+ if (timelineServiceEnabled) {
+ serviceTimelinePublisher.componentInstanceBecomeReady(containerSpec);
}
}
@@ -242,7 +286,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
@Override
public void transition(ComponentInstance compInstance,
ComponentInstanceEvent event) {
- compInstance.containerSpec.setState(ContainerState.RUNNING_BUT_UNREADY);
+ compInstance.setContainerState(ContainerState.RUNNING_BUT_UNREADY);
compInstance.component.decContainersReady(true);
}
}
@@ -276,11 +320,13 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
" completed. Reinsert back to pending list and requested ");
builder.append("a new container.").append(System.lineSeparator());
builder.append(" exitStatus=").append(
- failureBeforeLaunch ? null : event.getStatus().getExitStatus());
+ failureBeforeLaunch || event.getStatus() == null ? null :
+ event.getStatus().getExitStatus());
builder.append(", diagnostics=");
builder.append(failureBeforeLaunch ?
FAILED_BEFORE_LAUNCH_DIAG :
- event.getStatus().getDiagnostics());
+ (event.getStatus() != null ? event.getStatus().getDiagnostics() :
+ UPGRADE_FAILED));
if (event.getStatus() != null && event.getStatus().getExitStatus() != 0) {
LOG.error(builder.toString());
@@ -342,15 +388,14 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
ComponentInstanceEvent event) {
Component comp = compInstance.component;
+ ContainerStatus status = event.getStatus();
+ // status is not available when upgrade fails
String containerDiag = compInstance.getCompInstanceId() + ": " + (
- failedBeforeLaunching ?
- FAILED_BEFORE_LAUNCH_DIAG :
- event.getStatus().getDiagnostics());
+ failedBeforeLaunching ? FAILED_BEFORE_LAUNCH_DIAG :
+ (status != null ? status.getDiagnostics() : UPGRADE_FAILED));
compInstance.diagnostics.append(containerDiag + System.lineSeparator());
compInstance.cancelContainerStatusRetriever();
- if (compInstance.getState().equals(ComponentInstanceState.UPGRADING)) {
- compInstance.component.decContainersThatNeedUpgrade();
- }
+
if (compInstance.getState().equals(READY)) {
compInstance.component.decContainersReady(true);
}
@@ -387,10 +432,9 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
// record in ATS
compInstance.scheduler.getServiceTimelinePublisher()
.componentInstanceFinished(compInstance.getContainer().getId(),
- failedBeforeLaunching ?
- -1 :
- event.getStatus().getExitStatus(), ContainerState.FAILED,
- containerDiag);
+ failedBeforeLaunching || status == null ? -1 :
+ status.getExitStatus(),
+ ContainerState.FAILED, containerDiag);
// mark other component-instances/containers as STOPPED
for (ContainerId containerId : scheduler.getLiveInstances()
@@ -449,28 +493,129 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.equals(state) || ContainerState.SUCCEEDED.equals(state);
}
- private static class ContainerUpgradeTransition extends BaseTransition {
+ private static class StoppedAfterUpgradeTransition extends
+ BaseTransition {
@Override
- public void transition(ComponentInstance compInstance,
+ public void transition(ComponentInstance instance,
ComponentInstanceEvent event) {
- if (!compInstance.containerSpec.getState().equals(
- ContainerState.NEEDS_UPGRADE)) {
- //nothing to upgrade. this may happen with express upgrade.
+ instance.component.getUpgradeStatus().decContainersThatNeedUpgrade();
+ instance.component.decRunningContainers();
+
+ final ServiceScheduler scheduler = instance.component.getScheduler();
+ scheduler.getAmRMClient().releaseAssignedContainer(
+ event.getContainerId());
+ instance.scheduler.executorService.submit(
+ () -> instance.cleanupRegistry(event.getContainerId()));
+ scheduler.removeLiveCompInstance(event.getContainerId());
+ instance.component.getUpgradeStatus().containerFailedUpgrade();
+ instance.setContainerState(ContainerState.FAILED_UPGRADE);
+ instance.upgradeInProgress.set(false);
+ }
+ }
+
+ private static class StoppedAfterCancelUpgradeTransition implements
+ MultipleArcTransition<ComponentInstance, ComponentInstanceEvent,
+ ComponentInstanceState> {
+
+ private ContainerStoppedTransition stoppedTransition =
+ new ContainerStoppedTransition();
+
+ @Override
+ public ComponentInstanceState transition(ComponentInstance instance,
+ ComponentInstanceEvent event) {
+ if (instance.pendingCancelUpgrade) {
+ // cancellation of upgrade was triggered before the upgrade was
+ // finished.
+ LOG.info("{} received stopped but cancellation pending",
+ event.getContainerId());
+ instance.upgradeInProgress.set(true);
+ instance.cancelUpgrade();
+ instance.pendingCancelUpgrade = false;
+ return instance.getState();
+ }
+
+ // When upgrade is cancelled, and container re-init fails
+ instance.component.getCancelUpgradeStatus()
+ .decContainersThatNeedUpgrade();
+ instance.upgradeInProgress.set(false);
+ stoppedTransition.transition(instance, event);
+ return ComponentInstanceState.INIT;
+ }
+ }
+
+ private static class UpgradeTransition extends BaseTransition {
+
+ @Override
+ public void transition(ComponentInstance instance,
+ ComponentInstanceEvent event) {
+ if (!instance.component.getCancelUpgradeStatus().isCompleted()) {
+ // last check to see if cancellation was triggered. The component may
+ // have processed the cancel upgrade event but the instance doesn't know
+ // it yet. If cancellation has been triggered then no point in
+ // upgrading.
return;
}
- compInstance.containerSpec.setState(ContainerState.UPGRADING);
- compInstance.component.decContainersReady(false);
- ComponentEvent upgradeEvent = compInstance.component.getUpgradeEvent();
- compInstance.scheduler.getContainerLaunchService()
- .reInitCompInstance(compInstance.scheduler.getApp(), compInstance,
- compInstance.container,
- compInstance.component.createLaunchContext(
- upgradeEvent.getTargetSpec(),
- upgradeEvent.getUpgradeVersion()));
+ instance.upgradeInProgress.set(true);
+ instance.setContainerState(ContainerState.UPGRADING);
+ instance.component.decContainersReady(false);
+
+ Component.UpgradeStatus status = instance.component.getUpgradeStatus();
+ instance.scheduler.getContainerLaunchService()
+ .reInitCompInstance(instance.scheduler.getApp(), instance,
+ instance.container,
+ instance.component.createLaunchContext(
+ status.getTargetSpec(),
+ status.getTargetVersion()));
}
}
+ private static class CancelUpgradeTransition implements
+ MultipleArcTransition<ComponentInstance, ComponentInstanceEvent,
+ ComponentInstanceState> {
+
+ @Override
+ public ComponentInstanceState transition(ComponentInstance instance,
+ ComponentInstanceEvent event) {
+ if (instance.upgradeInProgress.compareAndSet(false, true)) {
+
+ Component.UpgradeStatus cancelStatus = instance.component
+ .getCancelUpgradeStatus();
+
+ if (instance.getServiceVersion().equals(
+ cancelStatus.getTargetVersion())) {
+ // previous upgrade didn't happen so just go back to READY
+ LOG.info("{} nothing to cancel", event.getContainerId());
+ cancelStatus.decContainersThatNeedUpgrade();
+ instance.setContainerState(ContainerState.READY);
+ ComponentEvent checkState = new ComponentEvent(
+ instance.component.getName(), ComponentEventType.CHECK_STABLE);
+ instance.scheduler.getDispatcher().getEventHandler()
+ .handle(checkState);
+ return ComponentInstanceState.READY;
+ } else {
+ instance.component.decContainersReady(false);
+ instance.cancelUpgrade();
+ }
+ } else {
+ LOG.info("{} pending cancellation", event.getContainerId());
+ instance.pendingCancelUpgrade = true;
+ }
+ return ComponentInstanceState.CANCEL_UPGRADING;
+ }
+ }
+
+ private void cancelUpgrade() {
+ LOG.info("{} cancelling upgrade", container.getId());
+ setContainerState(ContainerState.UPGRADING);
+ Component.UpgradeStatus cancelStatus = component.getCancelUpgradeStatus();
+ scheduler.getContainerLaunchService()
+ .reInitCompInstance(scheduler.getApp(), this,
+ this.container, this.component.createLaunchContext(
+ cancelStatus.getTargetSpec(),
+ cancelStatus.getTargetVersion()));
+ }
+
public ComponentInstanceState getState() {
this.readLock.lock();
@@ -505,6 +650,26 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
}
+ /**
+ * Sets the state of the container in the container spec. It is write
+ * protected.
+ *
+ * @param state container state
+ */
+ public void setContainerState(ContainerState state) {
+ this.writeLock.lock();
+ try {
+ ContainerState curState = containerSpec.getState();
+ if (!curState.equals(state)) {
+ containerSpec.setState(state);
+ LOG.info("{} spec state state changed from {} -> {}",
+ getCompInstanceId(), curState, state);
+ }
+ } finally {
+ this.writeLock.unlock();
+ }
+ }
+
@Override
public void handle(ComponentInstanceEvent event) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
index 665b8fa..b9181e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEventType.java
@@ -23,5 +23,6 @@ public enum ComponentInstanceEventType {
STOP,
BECOME_READY,
BECOME_NOT_READY,
- UPGRADE
+ UPGRADE,
+ CANCEL_UPGRADE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913f87da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
index f5de5cb..28cbcf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceState.java
@@ -22,5 +22,6 @@ public enum ComponentInstanceState {
INIT,
STARTED,
READY,
- UPGRADING
+ UPGRADING,
+ CANCEL_UPGRADING
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDDS-551. Fix the close container
status check in CloseContainerCommandHandler. Contributed by Shashikant
Banerjee.
Posted by in...@apache.org.
HDDS-551. Fix the close container status check in CloseContainerCommandHandler. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a5d4315
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a5d4315
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a5d4315
Branch: refs/heads/HDFS-13532
Commit: 2a5d4315bfe13c12cacc7718537077bf9abb22e2
Parents: 90e2e49
Author: Hanisha Koneru <ha...@apache.org>
Authored: Thu Sep 27 16:03:49 2018 -0700
Committer: Hanisha Koneru <ha...@apache.org>
Committed: Thu Sep 27 16:03:49 2018 -0700
----------------------------------------------------------------------
.../commandhandler/CloseContainerCommandHandler.java | 11 +++++++++++
.../hadoop/ozone/container/ozoneimpl/OzoneContainer.java | 9 ---------
2 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a5d4315/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index d4e13ee..0af591b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -74,6 +74,17 @@ public class CloseContainerCommandHandler implements CommandHandler {
CloseContainerCommandProto
.parseFrom(command.getProtoBufMessage());
containerID = closeContainerProto.getContainerID();
+ if (container.getContainerSet().getContainer(containerID)
+ .getContainerData().isClosed()) {
+ LOG.debug("Container {} is already closed", containerID);
+ // It might happen that the where the first attempt of closing the
+ // container failed with NOT_LEADER_EXCEPTION. In such cases, SCM will
+ // retry to check the container got really closed via Ratis.
+ // In such cases of the retry attempt, if the container is already
+ // closed via Ratis, we should just return.
+ cmdExecuted = true;
+ return;
+ }
HddsProtos.PipelineID pipelineID = closeContainerProto.getPipelineID();
HddsProtos.ReplicationType replicationType =
closeContainerProto.getReplicationType();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a5d4315/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index da58772..503112d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -188,15 +188,6 @@ public class OzoneContainer {
ContainerProtos.ContainerCommandRequestProto request,
ReplicationType replicationType,
PipelineID pipelineID) throws IOException {
- if (containerSet.getContainer(request.getContainerID())
- .getContainerData().isClosed()) {
- LOG.debug("Container {} is already closed", request.getContainerID());
- // It might happen that the where the first attempt of closing the
- // container failed with NOT_LEADER_EXCEPTION. In such cases, SCM will
- // retry to check the container got really closed via Ratis.
- // In such cases of the retry attempt, if the container is already closed
- // via Ratis, we should just return.
- }
LOG.info("submitting {} request over {} server for container {}",
request.getCmdType(), replicationType, request.getContainerID());
Preconditions.checkState(servers.containsKey(replicationType));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org