You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/12/18 00:38:46 UTC
[01/22] hadoop git commit: HDFS-14145.
TestBlockStorageMovementAttemptedItems#testNoBlockMovementAttemptFinishedReportAdded
fails sporadically in Trunk. Contributed by Ayush Saxena. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12943 59822e46d -> 861208dc8 (forced update)
HDFS-14145. TestBlockStorageMovementAttemptedItems#testNoBlockMovementAttemptFinishedReportAdded fails sporadically in Trunk. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb3382f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb3382f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb3382f6
Branch: refs/heads/HDFS-12943
Commit: cb3382f667a7f9c1f10f92e5c2fb315d72a4cec9
Parents: aa12859
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Dec 14 10:59:02 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Dec 14 10:59:02 2018 -0800
----------------------------------------------------------------------
.../server/namenode/sps/TestBlockStorageMovementAttemptedItems.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb3382f6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestBlockStorageMovementAttemptedItems.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestBlockStorageMovementAttemptedItems.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestBlockStorageMovementAttemptedItems.java
index f48521b..4437086 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestBlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestBlockStorageMovementAttemptedItems.java
@@ -117,7 +117,6 @@ public class TestBlockStorageMovementAttemptedItems {
*/
@Test(timeout = 30000)
public void testNoBlockMovementAttemptFinishedReportAdded() throws Exception {
- bsmAttemptedItems.start(); // start block movement report monitor thread
Long item = new Long(1234);
Block block = new Block(item);
DatanodeInfo dnInfo = DFSTestUtil.getLocalDatanodeInfo(9867);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/22] hadoop git commit: YARN-8963. Add flag to disable interactive
shell. Contributed by Eric Yang
Posted by sh...@apache.org.
YARN-8963. Add flag to disable interactive shell. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42297f7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42297f7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42297f7d
Branch: refs/heads/HDFS-12943
Commit: 42297f7d99df2769846a58efcf1b02fadb5ba60b
Parents: aab3109
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Dec 14 18:44:36 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Fri Dec 14 18:44:36 2018 -0800
----------------------------------------------------------------------
.../container-executor/impl/container-executor.c | 6 ++++++
.../container-executor/impl/container-executor.h | 6 ++++++
.../src/main/native/container-executor/impl/main.c | 14 ++++++++++----
.../src/site/markdown/SecureContainer.md | 3 +++
4 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42297f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index dcbfb0f..65bb803 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -73,6 +73,7 @@ static const int DEFAULT_MIN_USERID = 1000;
static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0};
+static const int DEFAULT_TERMINAL_SUPPORT_ENABLED = 0;
static const int DEFAULT_DOCKER_SUPPORT_ENABLED = 0;
static const int DEFAULT_TC_SUPPORT_ENABLED = 0;
static const int DEFAULT_MOUNT_CGROUP_SUPPORT_ENABLED = 0;
@@ -490,6 +491,11 @@ int is_feature_enabled(const char* feature_key, int default_value,
}
}
+int is_terminal_support_enabled() {
+ return is_feature_enabled(TERMINAL_SUPPORT_ENABLED_KEY,
+ DEFAULT_TERMINAL_SUPPORT_ENABLED, &executor_cfg);
+}
+
int is_docker_support_enabled() {
return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42297f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index bf99452..b215af7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -67,6 +67,7 @@ enum operations {
#define MIN_USERID_KEY "min.user.id"
#define BANNED_USERS_KEY "banned.users"
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
+#define TERMINAL_SUPPORT_ENABLED_KEY "feature.terminal.enabled"
#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
#define MOUNT_CGROUP_SUPPORT_ENABLED_KEY "feature.mount-cgroup.enabled"
@@ -341,3 +342,8 @@ char* flatten(char **args);
* Remove docker container
*/
int remove_docker_container(char **argv, int argc);
+
+/**
+ * Check if terminal feature is enabled
+ */
+int is_terminal_support_enabled();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42297f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 5124890..8cdb8a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -54,20 +54,26 @@ static void display_usage(FILE *stream) {
if(is_docker_support_enabled()) {
fprintf(stream,
" container-executor --run-docker <command-file>\n"
- " container-executor --exec-container <command-file>\n"
" container-executor --remove-docker-container [hierarchy] "
"<container_id>\n"
" container-executor --inspect-docker-container <container_id>\n");
} else {
fprintf(stream,
"[DISABLED] container-executor --run-docker <command-file>\n"
- "[DISABLED] container-executor --exec-container <command-file>\n"
"[DISABLED] container-executor --remove-docker-container [hierarchy] "
"<container_id>\n"
"[DISABLED] container-executor --inspect-docker-container "
"<format> ... <container_id>\n");
}
+ if (is_terminal_support_enabled()) {
+ fprintf(stream,
+ " container-executor --exec-container <command-file>\n");
+ } else {
+ fprintf(stream,
+ "[DISABLED] container-executor --exec-container <command-file>\n");
+ }
+
fprintf(stream,
" container-executor <user> <yarn-user> <command> <command-args>\n"
" where command and command-args: \n" \
@@ -351,7 +357,7 @@ static int validate_arguments(int argc, char **argv , int *operation) {
}
if (strcmp("--exec-container", argv[1]) == 0) {
- if(is_docker_support_enabled()) {
+ if(is_terminal_support_enabled()) {
if (argc != 3) {
display_usage(stdout);
return INVALID_ARGUMENT_NUMBER;
@@ -361,7 +367,7 @@ static int validate_arguments(int argc, char **argv , int *operation) {
*operation = EXEC_CONTAINER;
return 0;
} else {
- display_feature_disabled_message("docker");
+ display_feature_disabled_message("feature.terminal.enabled");
return FEATURE_DISABLED;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42297f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
index a66c042..397e76b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SecureContainer.md
@@ -59,8 +59,11 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema
banned.users=#comma separated list of users who can not run applications
allowed.system.users=#comma separated list of allowed system users
min.user.id=1000#Prevent other super-users
+feature.terminal.enabled=1
```
+Terminal feature (feature.terminal.enabled) allows restricted shell into secure container via YARN UI2.
+
###Windows Secure Container Executor (WSCE)
The Windows environment secure container executor is the `WindowsSecureContainerExecutor`. It uses the Windows S4U infrastructure to launch the container as the YARN application user. The WSCE requires the presense of the `hadoopwinutilsvc` service. This services is hosted by `%HADOOP_HOME%\bin\winutils.exe` started with the `service` command line argument. This service offers some privileged operations that require LocalSystem authority so that the NM is not required to run the entire JVM and all the NM code in an elevated context. The NM interacts with the `hadoopwintulsvc` service by means of Local RPC (LRPC) via calls JNI to the RCP client hosted in `hadoop.dll`.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/22] hadoop git commit: HDDS-908: NPE in TestOzoneRpcClient.
Contributed by Ajay Kumar.
Posted by sh...@apache.org.
HDDS-908: NPE in TestOzoneRpcClient.
Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54266538
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54266538
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54266538
Branch: refs/heads/HDFS-12943
Commit: 54266538192a558c6d80725c25912005090e14c4
Parents: 71e0b0d
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Dec 17 12:17:09 2018 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Dec 17 12:18:47 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/ozone/om/OzoneManager.java | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54266538/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a044bc2..f41eb88 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -1438,11 +1438,14 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
.setNodeType(HddsProtos.NodeType.DATANODE)
.setHostname(datanode.getHostName());
- dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
- .setType(ServicePort.Type.HTTP)
- .setValue(DatanodeDetails.getFromProtoBuf(datanode)
- .getPort(DatanodeDetails.Port.Name.REST).getValue())
- .build());
+ if(DatanodeDetails.getFromProtoBuf(datanode)
+ .getPort(DatanodeDetails.Port.Name.REST) != null) {
+ dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
+ .setType(ServicePort.Type.HTTP)
+ .setValue(DatanodeDetails.getFromProtoBuf(datanode)
+ .getPort(DatanodeDetails.Port.Name.REST).getValue())
+ .build());
+ }
services.add(dnServiceInfoBuilder.build());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/22] hadoop git commit: YARN-9040. Fixed memory leak in
LevelDBCacheTimelineStore and DBIterator. Contributed by Tarun Parimi
Posted by sh...@apache.org.
YARN-9040. Fixed memory leak in LevelDBCacheTimelineStore and DBIterator.
Contributed by Tarun Parimi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71e0b0d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71e0b0d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71e0b0d8
Branch: refs/heads/HDFS-12943
Commit: 71e0b0d8005ea1952dc7e582db15c2ac09df7c91
Parents: 346c0c8
Author: Eric Yang <ey...@apache.org>
Authored: Mon Dec 17 12:04:25 2018 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Dec 17 12:04:25 2018 -0500
----------------------------------------------------------------------
.../timeline/KeyValueBasedTimelineStore.java | 98 ++++++++++----------
.../server/timeline/MemoryTimelineStore.java | 37 +++++++-
.../timeline/TimelineStoreMapAdapter.java | 9 +-
.../timeline/LevelDBCacheTimelineStore.java | 14 ++-
4 files changed, 99 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71e0b0d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/KeyValueBasedTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/KeyValueBasedTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/KeyValueBasedTimelineStore.java
index 82db770..c24d904 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/KeyValueBasedTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/KeyValueBasedTimelineStore.java
@@ -42,7 +42,6 @@ import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -50,6 +49,7 @@ import java.util.Set;
import java.util.SortedSet;
import static org.apache.hadoop.yarn.server.timeline.TimelineDataManager.DEFAULT_DOMAIN_ID;
+import static org.apache.hadoop.yarn.server.timeline.TimelineStoreMapAdapter.CloseableIterator;
/**
* Map based implementation of {@link TimelineStore}. A hash map
@@ -114,66 +114,67 @@ abstract class KeyValueBasedTimelineStore
fields = EnumSet.allOf(Field.class);
}
- Iterator<TimelineEntity> entityIterator = null;
+ TimelineEntity firstEntity = null;
if (fromId != null) {
- TimelineEntity firstEntity = entities.get(new EntityIdentifier(fromId,
+ firstEntity = entities.get(new EntityIdentifier(fromId,
entityType));
if (firstEntity == null) {
return new TimelineEntities();
- } else {
- entityIterator = entities.valueSetIterator(firstEntity);
}
}
- if (entityIterator == null) {
- entityIterator = entities.valueSetIterator();
- }
List<TimelineEntity> entitiesSelected = new ArrayList<TimelineEntity>();
- while (entityIterator.hasNext()) {
- TimelineEntity entity = entityIterator.next();
- if (entitiesSelected.size() >= limit) {
- break;
- }
- if (!entity.getEntityType().equals(entityType)) {
- continue;
- }
- if (entity.getStartTime() <= windowStart) {
- continue;
- }
- if (entity.getStartTime() > windowEnd) {
- continue;
- }
- if (fromTs != null && entityInsertTimes.get(new EntityIdentifier(
- entity.getEntityId(), entity.getEntityType())) > fromTs) {
- continue;
- }
- if (primaryFilter != null &&
- !KeyValueBasedTimelineStoreUtils.matchPrimaryFilter(
- entity.getPrimaryFilters(), primaryFilter)) {
- continue;
- }
- if (secondaryFilters != null) { // AND logic
- boolean flag = true;
- for (NameValuePair secondaryFilter : secondaryFilters) {
- if (secondaryFilter != null && !KeyValueBasedTimelineStoreUtils
- .matchPrimaryFilter(entity.getPrimaryFilters(), secondaryFilter)
- && !KeyValueBasedTimelineStoreUtils.matchFilter(
- entity.getOtherInfo(), secondaryFilter)) {
- flag = false;
- break;
- }
+
+ try(CloseableIterator<TimelineEntity> entityIterator =
+ firstEntity == null ? entities.valueSetIterator() :
+ entities.valueSetIterator(firstEntity)) {
+ while (entityIterator.hasNext()) {
+ TimelineEntity entity = entityIterator.next();
+ if (entitiesSelected.size() >= limit) {
+ break;
}
- if (!flag) {
+ if (!entity.getEntityType().equals(entityType)) {
continue;
}
- }
- if (entity.getDomainId() == null) {
- entity.setDomainId(DEFAULT_DOMAIN_ID);
- }
- if (checkAcl == null || checkAcl.check(entity)) {
- entitiesSelected.add(entity);
+ if (entity.getStartTime() <= windowStart) {
+ continue;
+ }
+ if (entity.getStartTime() > windowEnd) {
+ continue;
+ }
+ if (fromTs != null && entityInsertTimes.get(
+ new EntityIdentifier(entity.getEntityId(), entity.getEntityType()))
+ > fromTs) {
+ continue;
+ }
+ if (primaryFilter != null && !KeyValueBasedTimelineStoreUtils
+ .matchPrimaryFilter(entity.getPrimaryFilters(), primaryFilter)) {
+ continue;
+ }
+ if (secondaryFilters != null) { // AND logic
+ boolean flag = true;
+ for (NameValuePair secondaryFilter : secondaryFilters) {
+ if (secondaryFilter != null && !KeyValueBasedTimelineStoreUtils
+ .matchPrimaryFilter(entity.getPrimaryFilters(), secondaryFilter)
+ && !KeyValueBasedTimelineStoreUtils
+ .matchFilter(entity.getOtherInfo(), secondaryFilter)) {
+ flag = false;
+ break;
+ }
+ }
+ if (!flag) {
+ continue;
+ }
+ }
+ if (entity.getDomainId() == null) {
+ entity.setDomainId(DEFAULT_DOMAIN_ID);
+ }
+ if (checkAcl == null || checkAcl.check(entity)) {
+ entitiesSelected.add(entity);
+ }
}
}
+
List<TimelineEntity> entitiesToReturn = new ArrayList<TimelineEntity>();
for (TimelineEntity entitySelected : entitiesSelected) {
entitiesToReturn.add(KeyValueBasedTimelineStoreUtils.maskFields(
@@ -569,6 +570,7 @@ abstract class KeyValueBasedTimelineStore
}
return o;
}
+
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71e0b0d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
index 5c2db00..f4aea45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.timeline;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -59,14 +60,15 @@ public class MemoryTimelineStore extends KeyValueBasedTimelineStore {
}
@Override
- public Iterator<V>
+ public CloseableIterator<V>
valueSetIterator() {
- return new TreeSet<>(internalMap.values()).iterator();
+ return wrapClosableIterator(new TreeSet<>(internalMap.values())
+ .iterator());
}
@Override
@SuppressWarnings("unchecked")
- public Iterator<V> valueSetIterator(V minV) {
+ public CloseableIterator<V> valueSetIterator(V minV) {
if (minV instanceof Comparable) {
TreeSet<V> tempTreeSet = new TreeSet<>();
for (V value : internalMap.values()) {
@@ -74,11 +76,38 @@ public class MemoryTimelineStore extends KeyValueBasedTimelineStore {
tempTreeSet.add(value);
}
}
- return tempTreeSet.iterator();
+ return wrapClosableIterator(tempTreeSet.iterator());
} else {
return valueSetIterator();
}
}
+
+ private CloseableIterator<V> wrapClosableIterator(
+ final Iterator<V> iterator) {
+ return new CloseableIterator<V>() {
+ private final Iterator<V> internalIterator = iterator;
+ @Override
+ public void close() throws IOException {
+ // Not implemented
+ }
+
+ @Override
+ public boolean hasNext() {
+ return internalIterator.hasNext();
+ }
+
+ @Override
+ public V next() {
+ return internalIterator.next();
+ }
+
+ @Override
+ public void remove() {
+ internalIterator.remove();
+ }
+ };
+
+ }
}
public MemoryTimelineStore() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71e0b0d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreMapAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreMapAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreMapAdapter.java
index 175ed0b..38fc28e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreMapAdapter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreMapAdapter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.timeline;
+import java.io.Closeable;
import java.util.Iterator;
/**
@@ -48,7 +49,7 @@ interface TimelineStoreMapAdapter<K, V> {
/**
* @return the iterator of the value set of the map
*/
- Iterator<V> valueSetIterator();
+ CloseableIterator<V> valueSetIterator();
/**
* Return the iterator of the value set of the map, starting from minV if type
@@ -56,5 +57,9 @@ interface TimelineStoreMapAdapter<K, V> {
* @param minV
* @return
*/
- Iterator<V> valueSetIterator(V minV);
+ CloseableIterator<V> valueSetIterator(V minV);
+
+ interface CloseableIterator<V> extends Iterator<V>, Closeable {}
}
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71e0b0d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
index f7a3d01..9b1ffdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LevelDBCacheTimelineStore.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.Iterator;
import java.util.Map;
/**
@@ -211,18 +210,18 @@ public class LevelDBCacheTimelineStore extends KeyValueBasedTimelineStore {
}
@Override
- public Iterator<V> valueSetIterator() {
+ public CloseableIterator<V> valueSetIterator() {
return getIterator(null, Long.MAX_VALUE);
}
@Override
- public Iterator<V> valueSetIterator(V minV) {
+ public CloseableIterator<V> valueSetIterator(V minV) {
return getIterator(
new EntityIdentifier(minV.getEntityId(), minV.getEntityType()),
minV.getStartTime());
}
- private Iterator<V> getIterator(
+ private CloseableIterator<V> getIterator(
EntityIdentifier startId, long startTimeMax) {
final DBIterator internalDbIterator = entityDb.iterator();
@@ -247,7 +246,7 @@ public class LevelDBCacheTimelineStore extends KeyValueBasedTimelineStore {
= entityPrefixKeyBuilder.getBytesForLookup();
internalDbIterator.seek(startPrefixBytes);
- return new Iterator<V>() {
+ return new CloseableIterator<V>() {
@Override
public boolean hasNext() {
if (!internalDbIterator.hasNext()) {
@@ -284,6 +283,11 @@ public class LevelDBCacheTimelineStore extends KeyValueBasedTimelineStore {
LOG.error("LevelDB map adapter does not support iterate-and-remove"
+ " use cases. ");
}
+
+ @Override
+ public void close() throws IOException {
+ internalDbIterator.close();
+ }
};
}
static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/22] hadoop git commit: HDFS-14116. [SBN read] Fix class cast
error in NNThroughputBenchmark with ObserverReadProxyProvider. Contributed by
Chao Sun.
Posted by sh...@apache.org.
HDFS-14116. [SBN read] Fix class cast error in NNThroughputBenchmark with ObserverReadProxyProvider. Contributed by Chao Sun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/861208dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/861208dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/861208dc
Branch: refs/heads/HDFS-12943
Commit: 861208dc8c14d94883c05caa2074aa4b90448ffe
Parents: 4d891a5
Author: Chao Sun <su...@apache.org>
Authored: Mon Dec 17 16:32:54 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Dec 17 16:35:57 2018 -0800
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/hdfs/DFSTestUtil.java | 12 +++++-------
.../hdfs/server/namenode/NNThroughputBenchmark.java | 6 ++++--
2 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/861208dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 97609fb..e3cab7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -71,7 +71,6 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@@ -2022,18 +2021,17 @@ public class DFSTestUtil {
* Get the RefreshUserMappingsProtocol RPC proxy for the NN associated with
* this DFSClient object
*
- * @param nameNodeUri the URI of the NN to get a proxy for.
+ * @param nnAddr the address of the NN to get a proxy for.
*
* @return the RefreshUserMappingsProtocol RPC proxy associated with this
* DFSClient object
*/
@VisibleForTesting
public static RefreshUserMappingsProtocol getRefreshUserMappingsProtocolProxy(
- Configuration conf, URI nameNodeUri) throws IOException {
- final AtomicBoolean nnFallbackToSimpleAuth = new AtomicBoolean(false);
- return NameNodeProxies.createProxy(conf,
- nameNodeUri, RefreshUserMappingsProtocol.class,
- nnFallbackToSimpleAuth).getProxy();
+ Configuration conf, InetSocketAddress nnAddr) throws IOException {
+ return NameNodeProxies.createNonHAProxy(
+ conf, nnAddr, RefreshUserMappingsProtocol.class,
+ UserGroupInformation.getCurrentUser(), false).getProxy();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/861208dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index a9c4578..654a8a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
@@ -1519,10 +1520,11 @@ public class NNThroughputBenchmark implements Tool {
nameNodeProto = DFSTestUtil.getNamenodeProtocolProxy(config, nnUri,
UserGroupInformation.getCurrentUser());
clientProto = dfs.getClient().getNamenode();
+ InetSocketAddress nnAddr = DFSUtilClient.getNNAddress(nnUri);
dataNodeProto = new DatanodeProtocolClientSideTranslatorPB(
- DFSUtilClient.getNNAddress(nnUri), config);
+ nnAddr, config);
refreshUserMappingsProto =
- DFSTestUtil.getRefreshUserMappingsProtocolProxy(config, nnUri);
+ DFSTestUtil.getRefreshUserMappingsProtocolProxy(config, nnAddr);
getBlockPoolId(dfs);
}
// run each benchmark
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/22] hadoop git commit: YARN-9095. Removed Unused field from
Resource: NUM_MANDATORY_RESOURCES
Posted by sh...@apache.org.
YARN-9095. Removed Unused field from Resource: NUM_MANDATORY_RESOURCES
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04c03476
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04c03476
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04c03476
Branch: refs/heads/HDFS-12943
Commit: 04c03476c9724f44a9c767db8e4af812a3669221
Parents: 4f62233
Author: Vidura Mudalige <vi...@kohls.com>
Authored: Wed Dec 12 12:10:02 2018 +0530
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Sat Dec 15 16:07:27 2018 +0900
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/api/records/Resource.java | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04c03476/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 7740354..b55a848 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -66,11 +66,6 @@ public abstract class Resource implements Comparable<Resource> {
protected ResourceInformation[] resources = null;
- // Number of mandatory resources, this is added to avoid invoke
- // MandatoryResources.values().length, since values() internally will
- // copy array, etc.
- protected static final int NUM_MANDATORY_RESOURCES = 2;
-
@Private
public static final int MEMORY_INDEX = 0;
@Private
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/22] hadoop git commit: HDFS-14146. [SBN read] Handle exceptions
from and prevent handler threads from blocking within internalQueueCall.
Contributed by Chao Sun.
Posted by sh...@apache.org.
HDFS-14146. [SBN read] Handle exceptions from and prevent handler threads from blocking within internalQueueCall. Contributed by Chao Sun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f19f4cb8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f19f4cb8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f19f4cb8
Branch: refs/heads/HDFS-12943
Commit: f19f4cb8ff4b212dc5c491e5560b14ac59cfc63b
Parents: 5b10b80
Author: Erik Krogen <xk...@apache.org>
Authored: Fri Dec 14 14:02:20 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Dec 17 16:02:08 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/ipc/ExternalCall.java | 3 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 39 +++++++++---
.../ha/TestConsistentReadsObserver.java | 63 ++++++++++++++++++++
3 files changed, 97 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f19f4cb8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java
index 5566136..5cc3665 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ExternalCall.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.ipc.Server.Call;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto;
import org.apache.hadoop.security.UserGroupInformation;
public abstract class ExternalCall<T> extends Call {
@@ -78,7 +79,7 @@ public abstract class ExternalCall<T> extends Call {
}
@Override
- final void doResponse(Throwable t) {
+ final void doResponse(Throwable t, RpcStatusProto status) {
synchronized(done) {
error = t;
done.set(true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f19f4cb8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index bab2893..dca9551 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -820,7 +820,11 @@ public abstract class Server {
}
}
- void doResponse(Throwable t) throws IOException {}
+ void doResponse(Throwable t) throws IOException {
+ doResponse(t, RpcStatusProto.FATAL);
+ }
+
+ void doResponse(Throwable t, RpcStatusProto proto) throws IOException {}
// For Schedulable
@Override
@@ -987,15 +991,17 @@ public abstract class Server {
}
@Override
- void doResponse(Throwable t) throws IOException {
+ void doResponse(Throwable t, RpcStatusProto status) throws IOException {
RpcCall call = this;
if (t != null) {
+ if (status == null) {
+ status = RpcStatusProto.FATAL;
+ }
// clone the call to prevent a race with another thread stomping
// on the response while being sent. the original call is
// effectively discarded since the wait count won't hit zero
call = new RpcCall(this);
- setupResponse(call,
- RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
+ setupResponse(call, status, RpcErrorCodeProto.ERROR_RPC_SERVER,
null, t.getClass().getName(), StringUtils.stringifyException(t));
} else {
setupResponse(call, call.responseParams.returnStatus,
@@ -2749,8 +2755,18 @@ public abstract class Server {
private void internalQueueCall(Call call)
throws IOException, InterruptedException {
+ internalQueueCall(call, true);
+ }
+
+ private void internalQueueCall(Call call, boolean blocking)
+ throws IOException, InterruptedException {
try {
- callQueue.put(call); // queue the call; maybe blocked here
+ // queue the call, may be blocked if blocking is true.
+ if (blocking) {
+ callQueue.put(call);
+ } else {
+ callQueue.add(call);
+ }
} catch (CallQueueOverflowException cqe) {
// If rpc scheduler indicates back off based on performance degradation
// such as response time or rpc queue is full, we will ask the client
@@ -2794,8 +2810,8 @@ public abstract class Server {
* In case of Observer, it handles only reads, which are
* commutative.
*/
- //Re-queue the call and continue
- internalQueueCall(call);
+ // Re-queue the call and continue
+ requeueCall(call);
continue;
}
if (LOG.isDebugEnabled()) {
@@ -2837,6 +2853,15 @@ public abstract class Server {
LOG.debug(Thread.currentThread().getName() + ": exiting");
}
+ private void requeueCall(Call call)
+ throws IOException, InterruptedException {
+ try {
+ internalQueueCall(call, false);
+ } catch (RpcServerException rse) {
+ call.doResponse(rse.getCause(), rse.getRpcStatusProto());
+ }
+ }
+
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f19f4cb8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
index e1fadaf..fe5345d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
@@ -25,12 +25,16 @@ import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.ipc.RpcScheduler;
+import org.apache.hadoop.ipc.Schedulable;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.After;
@@ -85,6 +89,36 @@ public class TestConsistentReadsObserver {
}
@Test
+ public void testRequeueCall() throws Exception {
+ setObserverRead(true);
+
+ // Update the configuration just for the observer, by enabling
+ // IPC backoff and using the test scheduler class, which starts to backoff
+ // after certain number of calls.
+ final int observerIdx = 2;
+ NameNode nn = dfsCluster.getNameNode(observerIdx);
+ int port = nn.getNameNodeAddress().getPort();
+ Configuration configuration = dfsCluster.getConfiguration(observerIdx);
+ String prefix = CommonConfigurationKeys.IPC_NAMESPACE + "." + port + ".";
+ configuration.set(prefix + CommonConfigurationKeys.IPC_SCHEDULER_IMPL_KEY,
+ TestRpcScheduler.class.getName());
+ configuration.setBoolean(prefix
+ + CommonConfigurationKeys.IPC_BACKOFF_ENABLE, true);
+
+ dfsCluster.restartNameNode(observerIdx);
+ dfsCluster.transitionToObserver(observerIdx);
+
+ dfs.create(testPath, (short)1).close();
+ assertSentTo(0);
+
+ // Since we haven't tailed edit logs on the observer, it will fall behind
+ // and keep re-queueing the incoming request. Eventually, RPC backoff will
+ // be triggered and client should retry active NN.
+ dfs.getFileStatus(testPath);
+ assertSentTo(0);
+ }
+
+ @Test
public void testMsyncSimple() throws Exception {
// 0 == not completed, 1 == succeeded, -1 == failed
AtomicInteger readStatus = new AtomicInteger(0);
@@ -169,4 +203,33 @@ public class TestConsistentReadsObserver {
dfs = HATestUtil.configureObserverReadFs(
dfsCluster, conf, ObserverReadProxyProvider.class, flag);
}
+
+ /**
+ * A dummy test scheduler that starts backoff after a fixed number
+ * of requests.
+ */
+ public static class TestRpcScheduler implements RpcScheduler {
+ // Allow a number of RPCs to pass in order for the NN restart to succeed.
+ private int allowed = 10;
+ public TestRpcScheduler() {}
+
+ @Override
+ public int getPriorityLevel(Schedulable obj) {
+ return 0;
+ }
+
+ @Override
+ public boolean shouldBackOff(Schedulable obj) {
+ return --allowed < 0;
+ }
+
+ @Override
+ public void addResponseTime(String name, int priorityLevel, int queueTime,
+ int processingTime) {
+ }
+
+ @Override
+ public void stop() {
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/22] hadoop git commit: YARN-9032. Support sh shell for
interactive container shell at command line. Contributed by Eric Yang
Posted by sh...@apache.org.
YARN-9032. Support sh shell for interactive container shell at command line. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca90b248
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca90b248
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca90b248
Branch: refs/heads/HDFS-12943
Commit: ca90b2480ecb9a627914e7a463b6293024ad6e4d
Parents: 00d5e63
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Dec 14 14:08:22 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Fri Dec 14 14:08:22 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/client/api/impl/YarnClientImpl.java | 5 +++--
.../apache/hadoop/yarn/client/cli/ApplicationCLI.java | 5 +++--
.../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../linux/runtime/DefaultLinuxContainerRuntime.java | 5 ++++-
.../linux/runtime/DockerLinuxContainerRuntime.java | 6 +++++-
.../nodemanager/executor/ContainerExecContext.java | 14 +++++++++++++-
.../nodemanager/webapp/ContainerShellWebSocket.java | 10 ++++++++++
7 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca90b248/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index f920ebb..a7487f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -1104,10 +1104,11 @@ public class YarnClientImpl extends YarnClient {
}
WebSocketClient client = new WebSocketClient();
URI uri = URI.create(protocol + host + ":" + port + "/container/" +
- containerId);
+ containerId + "/" + command);
if (!UserGroupInformation.isSecurityEnabled()) {
uri = URI.create(protocol + host + ":" + port + "/container/" +
- containerId + "?user.name=" + System.getProperty("user.name"));
+ containerId + "/" + command + "?user.name=" +
+ System.getProperty("user.name"));
}
try {
client.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca90b248/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index ff8142a..be3c7db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -335,7 +335,8 @@ public class ApplicationCLI extends YarnCLI {
"app version, -components to filter instances based on component " +
"names, -states to filter instances based on instance state.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
- opts.getOption(SHELL_CMD).setArgName("Container ID");
+ opts.getOption(SHELL_CMD).setArgName("Container ID [bash|sh]");
+ opts.getOption(SHELL_CMD).setArgs(3);
opts.getOption(STATUS_CMD).setArgName("Container ID");
opts.getOption(LIST_CMD).setArgName("Application Name or Attempt ID");
opts.addOption(APP_TYPE_CMD, true, "Works with -list to " +
@@ -584,7 +585,7 @@ public class ApplicationCLI extends YarnCLI {
ShellContainerCommand command =
ShellContainerCommand.BASH;
if (shellArgs.length == 2) {
- command = ShellContainerCommand.valueOf(shellArgs[1]);
+ command = ShellContainerCommand.valueOf(shellArgs[1].toUpperCase());
}
shellToContainer(containerId, command);
} else if (cliParser.hasOption(LAUNCH_CMD)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca90b248/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 8c59a61..a58855c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -2314,7 +2314,7 @@ public class TestYarnCLI {
pw.println(" -components <arg> Works with -list to filter instances based on input comma-separated list of component names.");
pw.println(" -help Displays help for all commands.");
pw.println(" -list <Application Name or Attempt ID> List containers for application attempt when application attempt ID is provided. When application name is provided, then it finds the instances of the application based on app's own implementation, and -appTypes option must be specified unless it is the default yarn-service type. With app name, it supports optional use of -version to filter instances based on app version, -components to filter instances based on component names, -states to filter instances based on instance state.");
- pw.println(" -shell <Container ID> Run a shell in the container.");
+ pw.println(" -shell <Container ID [bash|sh]> Run a shell in the container.");
pw.println(" -signal <container ID [signal command]> Signal the container.");
pw.println("The available signal commands are ");
pw.println(java.util.Arrays.asList(SignalContainerCommand.values()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca90b248/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 8b00d4c..f7dffc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -267,7 +267,10 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
cmd.put("user", user);
// launch-command = bash,-i
List<String> commands = new ArrayList<String>();
- commands.add("/bin/bash");
+ StringBuilder sb = new StringBuilder();
+ sb.append("/bin/");
+ sb.append(ctx.getShell());
+ commands.add(sb.toString());
commands.add("-ir");
cmd.put("launch-command", commands);
// workdir = ../nm-local-dir/usercache/appcache/appid/containerid
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca90b248/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 225bc19..99a8a66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1178,7 +1178,11 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
dockerExecCommand.setInteractive();
dockerExecCommand.setTTY();
List<String> command = new ArrayList<String>();
- command.add("bash");
+ StringBuilder sb = new StringBuilder();
+ sb.append("/bin/");
+ sb.append(ctx.getShell());
+ command.add(sb.toString());
+ command.add("-i");
dockerExecCommand.setOverrideCommandWithArgs(command);
String commandFile = dockerClient.writeCommandToTempFile(dockerExecCommand,
ContainerId.fromString(containerId), nmContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca90b248/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerExecContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerExecContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerExecContext.java
index 0a2228f..1e0cfda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerExecContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerExecContext.java
@@ -35,6 +35,7 @@ public final class ContainerExecContext {
private final String user;
private final String appId;
private final Container container;
+ private String command;
private final LocalDirsHandlerService localDirsHandler;
/**
@@ -44,6 +45,7 @@ public final class ContainerExecContext {
private String user;
private String appId;
private Container container;
+ private String command;
private LocalDirsHandlerService localDirsHandler;
public Builder() {
@@ -73,12 +75,18 @@ public final class ContainerExecContext {
this.localDirsHandler = ldhs;
return this;
}
+
+ public Builder setShell(String command) {
+ this.command = command;
+ return this;
+ }
}
private ContainerExecContext(Builder builder) {
- this.container = builder.container;
this.user = builder.user;
this.appId = builder.appId;
+ this.container = builder.container;
+ this.command = builder.command;
this.localDirsHandler = builder.localDirsHandler;
}
@@ -94,6 +102,10 @@ public final class ContainerExecContext {
return this.container;
}
+ public String getShell() {
+ return this.command;
+ }
+
public LocalDirsHandlerService getLocalDirsHandlerService() {
return this.localDirsHandler;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca90b248/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
index b1c389b..2fc1d16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ShellContainerCommand;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -98,8 +99,16 @@ public class ContainerShellWebSocket {
public void onConnect(Session session) {
try {
URI containerURI = session.getUpgradeRequest().getRequestURI();
+ String command = "bash";
String[] containerPath = containerURI.getPath().split("/");
String cId = containerPath[2];
+ if (containerPath.length==4) {
+ for (ShellContainerCommand c : ShellContainerCommand.values()) {
+ if (c.name().equalsIgnoreCase(containerPath[3])) {
+ command = containerPath[3].toLowerCase();
+ }
+ }
+ }
Container container = nmContext.getContainers().get(ContainerId
.fromString(cId));
if (!checkAuthorization(session, container)) {
@@ -113,6 +122,7 @@ public class ContainerShellWebSocket {
.Builder()
.setContainer(container)
.setNMLocalPath(nmContext.getLocalDirsHandler())
+ .setShell(command)
.build();
pair = exec.execContainer(execContext);
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/22] hadoop git commit: HDDS-99. Adding SCM Audit log. Contributed
by Dinesh Chitlangia.
Posted by sh...@apache.org.
HDDS-99. Adding SCM Audit log. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94b368f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94b368f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94b368f2
Branch: refs/heads/HDFS-12943
Commit: 94b368f29fb5286253f4e5cac2d30b61cb62a7e5
Parents: 5426653
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Mon Dec 17 15:40:22 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Mon Dec 17 15:40:22 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/ozone/audit/SCMAction.java | 45 ++++
.../apache/hadoop/ozone/common/BlockGroup.java | 8 +
.../hdds/scm/server/SCMBlockProtocolServer.java | 106 ++++++++-
.../scm/server/SCMClientProtocolServer.java | 227 +++++++++++++++----
.../scm/server/SCMDatanodeProtocolServer.java | 111 ++++++++-
hadoop-ozone/common/src/main/bin/ozone | 1 +
.../dist/dev-support/bin/dist-layout-stitching | 1 +
.../src/main/conf/scm-audit-log4j2.properties | 90 ++++++++
8 files changed, 533 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
new file mode 100644
index 0000000..950a783
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.audit;
+
+/**
+ * Enum to define Audit Action types for SCM.
+ */
+public enum SCMAction implements AuditAction {
+
+ GET_VERSION,
+ REGISTER,
+ SEND_HEARTBEAT,
+ GET_SCM_INFO,
+ ALLOCATE_BLOCK,
+ DELETE_KEY_BLOCK,
+ ALLOCATE_CONTAINER,
+ GET_CONTAINER,
+ GET_CONTAINER_WITH_PIPELINE,
+ LIST_CONTAINER,
+ LIST_PIPELINE,
+ CLOSE_PIPELINE,
+ DELETE_CONTAINER,
+ IN_CHILL_MODE,
+ FORCE_EXIT_CHILL_MODE;
+
+ @Override
+ public String getAction() {
+ return this.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
index 2f6030f..1925c22 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
@@ -72,6 +72,14 @@ public final class BlockGroup {
return new Builder();
}
+ @Override
+ public String toString() {
+ return "BlockGroup[" +
+ "groupID='" + groupID + '\'' +
+ ", blockIDs=" + blockIDs +
+ ']';
+ }
+
/**
* BlockGroup instance builder.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index 3bb284e..26e3b13 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -21,6 +21,7 @@
*/
package org.apache.hadoop.hdds.scm.server;
+import com.google.common.collect.Maps;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -35,6 +36,14 @@ import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.audit.AuditAction;
+import org.apache.hadoop.ozone.audit.AuditEventStatus;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.audit.AuditMessage;
+import org.apache.hadoop.ozone.audit.Auditor;
+import org.apache.hadoop.ozone.audit.SCMAction;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
@@ -47,6 +56,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys
.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
@@ -62,10 +72,14 @@ import static org.apache.hadoop.hdds.scm.server.StorageContainerManager
* SCM block protocol is the protocol used by Namenode and OzoneManager to get
* blocks from the SCM.
*/
-public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
+public class SCMBlockProtocolServer implements
+ ScmBlockLocationProtocol, Auditor {
private static final Logger LOG =
LoggerFactory.getLogger(SCMBlockProtocolServer.class);
+ private static final AuditLogger AUDIT =
+ new AuditLogger(AuditLoggerType.SCMLOGGER);
+
private final StorageContainerManager scm;
private final OzoneConfiguration conf;
private final RPC.Server blockRpcServer;
@@ -140,7 +154,27 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
public AllocatedBlock allocateBlock(long size, HddsProtos.ReplicationType
type, HddsProtos.ReplicationFactor factor, String owner) throws
IOException {
- return scm.getScmBlockManager().allocateBlock(size, type, factor, owner);
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("size", String.valueOf(size));
+ auditMap.put("type", type.name());
+ auditMap.put("factor", factor.name());
+ auditMap.put("owner", owner);
+ boolean auditSuccess = true;
+ try {
+ return scm.getScmBlockManager().allocateBlock(size, type, factor, owner);
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logWriteFailure(
+ buildAuditMessageForFailure(SCMAction.ALLOCATE_BLOCK, auditMap, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.ALLOCATE_BLOCK, auditMap)
+ );
+ }
+ }
}
/**
@@ -155,17 +189,26 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
LOG.info("SCM is informed by OM to delete {} blocks", keyBlocksInfoList
.size());
List<DeleteBlockGroupResult> results = new ArrayList<>();
+ Map<String, String> auditMap = Maps.newHashMap();
for (BlockGroup keyBlocks : keyBlocksInfoList) {
ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result resultCode;
try {
// We delete blocks in an atomic operation to prevent getting
// into state like only a partial of blocks are deleted,
// which will leave key in an inconsistent state.
+ auditMap.put("keyBlockToDelete", keyBlocks.toString());
scm.getScmBlockManager().deleteBlocks(keyBlocks.getBlockIDList());
resultCode = ScmBlockLocationProtocolProtos.DeleteScmBlockResult
.Result.success;
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.DELETE_KEY_BLOCK, auditMap)
+ );
} catch (SCMException scmEx) {
LOG.warn("Fail to delete block: {}", keyBlocks.getGroupID(), scmEx);
+ AUDIT.logWriteFailure(
+ buildAuditMessageForFailure(SCMAction.DELETE_KEY_BLOCK, auditMap,
+ scmEx)
+ );
switch (scmEx.getResult()) {
case CHILL_MODE_EXCEPTION:
resultCode = ScmBlockLocationProtocolProtos.DeleteScmBlockResult
@@ -182,6 +225,10 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
} catch (IOException ex) {
LOG.warn("Fail to delete blocks for object key: {}", keyBlocks
.getGroupID(), ex);
+ AUDIT.logWriteFailure(
+ buildAuditMessageForFailure(SCMAction.DELETE_KEY_BLOCK, auditMap,
+ ex)
+ );
resultCode = ScmBlockLocationProtocolProtos.DeleteScmBlockResult
.Result.unknownFailure;
}
@@ -197,10 +244,55 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
@Override
public ScmInfo getScmInfo() throws IOException {
- ScmInfo.Builder builder =
- new ScmInfo.Builder()
- .setClusterId(scm.getScmStorage().getClusterID())
- .setScmId(scm.getScmStorage().getScmId());
- return builder.build();
+ boolean auditSuccess = true;
+ try{
+ ScmInfo.Builder builder =
+ new ScmInfo.Builder()
+ .setClusterId(scm.getScmStorage().getClusterID())
+ .setScmId(scm.getScmStorage().getScmId());
+ return builder.build();
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logReadFailure(
+ buildAuditMessageForFailure(SCMAction.GET_SCM_INFO, null, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.GET_SCM_INFO, null)
+ );
+ }
+ }
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForSuccess(
+ AuditAction op, Map<String, String> auditMap) {
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.SUCCESS.toString())
+ .withException(null)
+ .build();
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForFailure(AuditAction op, Map<String,
+ String> auditMap, Throwable throwable) {
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.FAILURE.toString())
+ .withException(throwable)
+ .build();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index d80d6e2..998512c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -23,6 +23,7 @@ package org.apache.hadoop.hdds.scm.server;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -52,6 +53,14 @@ import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.audit.AuditAction;
+import org.apache.hadoop.ozone.audit.AuditEventStatus;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.audit.AuditMessage;
+import org.apache.hadoop.ozone.audit.Auditor;
+import org.apache.hadoop.ozone.audit.SCMAction;
import org.apache.hadoop.ozone.protocolPB
.StorageContainerLocationProtocolServerSideTranslatorPB;
import org.apache.hadoop.security.UserGroupInformation;
@@ -62,6 +71,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
@@ -84,9 +94,11 @@ import static org.apache.hadoop.hdds.scm.server.StorageContainerManager
* The RPC server that listens to requests from clients.
*/
public class SCMClientProtocolServer implements
- StorageContainerLocationProtocol, EventHandler<Boolean> {
+ StorageContainerLocationProtocol, EventHandler<Boolean>, Auditor {
private static final Logger LOG =
LoggerFactory.getLogger(SCMClientProtocolServer.class);
+ private static final AuditLogger AUDIT =
+ new AuditLogger(AuditLoggerType.SCMLOGGER);
private final RPC.Server clientRpcServer;
private final InetSocketAddress clientRpcAddress;
private final StorageContainerManager scm;
@@ -177,46 +189,84 @@ public class SCMClientProtocolServer implements
@Override
public ContainerInfo getContainer(long containerID) throws IOException {
String remoteUser = getRpcRemoteUsername();
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("containerID", String.valueOf(containerID));
getScm().checkAdminAccess(remoteUser);
- return scm.getContainerManager()
- .getContainer(ContainerID.valueof(containerID));
+ try {
+ return scm.getContainerManager()
+ .getContainer(ContainerID.valueof(containerID));
+ } catch (IOException ex) {
+ auditSuccess = false;
+ AUDIT.logReadFailure(
+ buildAuditMessageForFailure(SCMAction.GET_CONTAINER, auditMap, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.GET_CONTAINER, auditMap)
+ );
+ }
+ }
+
}
@Override
public ContainerWithPipeline getContainerWithPipeline(long containerID)
throws IOException {
- if (chillModePrecheck.isInChillMode()) {
- ContainerInfo contInfo = scm.getContainerManager()
- .getContainer(ContainerID.valueof(containerID));
- if (contInfo.isOpen()) {
- if (!hasRequiredReplicas(contInfo)) {
- throw new SCMException("Open container " + containerID + " doesn't"
- + " have enough replicas to service this operation in "
- + "Chill mode.", ResultCodes.CHILL_MODE_EXCEPTION);
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("containerID", String.valueOf(containerID));
+ boolean auditSuccess = true;
+ try {
+ if (chillModePrecheck.isInChillMode()) {
+ ContainerInfo contInfo = scm.getContainerManager()
+ .getContainer(ContainerID.valueof(containerID));
+ if (contInfo.isOpen()) {
+ if (!hasRequiredReplicas(contInfo)) {
+ throw new SCMException("Open container " + containerID + " doesn't"
+ + " have enough replicas to service this operation in "
+ + "Chill mode.", ResultCodes.CHILL_MODE_EXCEPTION);
+ }
}
}
- }
- getScm().checkAdminAccess(null);
-
- final ContainerID id = ContainerID.valueof(containerID);
- final ContainerInfo container = scm.getContainerManager().getContainer(id);
- final Pipeline pipeline;
-
- if (container.isOpen()) {
- // Ratis pipeline
- pipeline = scm.getPipelineManager()
- .getPipeline(container.getPipelineID());
- } else {
- pipeline = scm.getPipelineManager().createPipeline(
- HddsProtos.ReplicationType.STAND_ALONE,
- container.getReplicationFactor(),
- scm.getContainerManager()
- .getContainerReplicas(id).stream()
- .map(ContainerReplica::getDatanodeDetails)
- .collect(Collectors.toList()));
- }
+ getScm().checkAdminAccess(null);
+
+ final ContainerID id = ContainerID.valueof(containerID);
+ final ContainerInfo container = scm.getContainerManager().
+ getContainer(id);
+ final Pipeline pipeline;
+
+ if (container.isOpen()) {
+ // Ratis pipeline
+ pipeline = scm.getPipelineManager()
+ .getPipeline(container.getPipelineID());
+ } else {
+ pipeline = scm.getPipelineManager().createPipeline(
+ HddsProtos.ReplicationType.STAND_ALONE,
+ container.getReplicationFactor(),
+ scm.getContainerManager()
+ .getContainerReplicas(id).stream()
+ .map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList()));
+ }
- return new ContainerWithPipeline(container, pipeline);
+ return new ContainerWithPipeline(container, pipeline);
+ } catch (IOException ex) {
+ auditSuccess = false;
+ AUDIT.logReadFailure(
+ buildAuditMessageForFailure(SCMAction.GET_CONTAINER_WITH_PIPELINE,
+ auditMap, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.GET_CONTAINER_WITH_PIPELINE,
+ auditMap)
+ );
+ }
+ }
}
/**
@@ -238,16 +288,51 @@ public class SCMClientProtocolServer implements
@Override
public List<ContainerInfo> listContainer(long startContainerID,
int count) throws IOException {
- return scm.getContainerManager().
- listContainer(ContainerID.valueof(startContainerID), count);
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("startContainerID", String.valueOf(startContainerID));
+ auditMap.put("count", String.valueOf(count));
+ try {
+ return scm.getContainerManager().
+ listContainer(ContainerID.valueof(startContainerID), count);
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logReadFailure(
+ buildAuditMessageForFailure(SCMAction.LIST_CONTAINER, auditMap, ex));
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.LIST_CONTAINER, auditMap));
+ }
+ }
+
}
@Override
public void deleteContainer(long containerID) throws IOException {
String remoteUser = getRpcRemoteUsername();
- getScm().checkAdminAccess(remoteUser);
- scm.getContainerManager().deleteContainer(ContainerID.valueof(containerID));
-
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("containerID", String.valueOf(containerID));
+ auditMap.put("remoteUser", remoteUser);
+ try {
+ getScm().checkAdminAccess(remoteUser);
+ scm.getContainerManager().deleteContainer(
+ ContainerID.valueof(containerID));
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logWriteFailure(
+ buildAuditMessageForFailure(SCMAction.DELETE_CONTAINER, auditMap, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.DELETE_CONTAINER, auditMap)
+ );
+ }
+ }
}
@Override
@@ -311,26 +396,48 @@ public class SCMClientProtocolServer implements
@Override
public List<Pipeline> listPipelines() {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.LIST_PIPELINE, null));
return scm.getPipelineManager().getPipelines();
}
@Override
public void closePipeline(HddsProtos.PipelineID pipelineID)
throws IOException {
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("pipelineID", pipelineID.getId());
PipelineManager pipelineManager = scm.getPipelineManager();
Pipeline pipeline =
pipelineManager.getPipeline(PipelineID.getFromProtobuf(pipelineID));
RatisPipelineUtils
.finalizeAndDestroyPipeline(pipelineManager, pipeline, conf, false);
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.CLOSE_PIPELINE, null)
+ );
}
@Override
public ScmInfo getScmInfo() throws IOException {
- ScmInfo.Builder builder =
- new ScmInfo.Builder()
- .setClusterId(scm.getScmStorage().getClusterID())
- .setScmId(scm.getScmStorage().getScmId());
- return builder.build();
+ boolean auditSuccess = true;
+ try{
+ ScmInfo.Builder builder =
+ new ScmInfo.Builder()
+ .setClusterId(scm.getScmStorage().getClusterID())
+ .setScmId(scm.getScmStorage().getScmId());
+ return builder.build();
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logReadFailure(
+ buildAuditMessageForFailure(SCMAction.GET_SCM_INFO, null, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.GET_SCM_INFO, null)
+ );
+ }
+ }
}
/**
@@ -341,6 +448,9 @@ public class SCMClientProtocolServer implements
*/
@Override
public boolean inChillMode() throws IOException {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.IN_CHILL_MODE, null)
+ );
return scm.isInChillMode();
}
@@ -352,6 +462,9 @@ public class SCMClientProtocolServer implements
*/
@Override
public boolean forceExitChillMode() throws IOException {
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.FORCE_EXIT_CHILL_MODE, null)
+ );
return scm.exitChillMode();
}
@@ -409,4 +522,34 @@ public class SCMClientProtocolServer implements
}
return returnSet;
}
+
+ @Override
+ public AuditMessage buildAuditMessageForSuccess(
+ AuditAction op, Map<String, String> auditMap) {
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.SUCCESS.toString())
+ .withException(null)
+ .build();
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForFailure(AuditAction op, Map<String,
+ String> auditMap, Throwable throwable) {
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.FAILURE.toString())
+ .withException(throwable)
+ .build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 77ef9c8..6a3552e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -23,6 +23,7 @@ package org.apache.hadoop.hdds.scm.server;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -75,6 +76,14 @@ import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.audit.AuditAction;
+import org.apache.hadoop.ozone.audit.AuditEventStatus;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.audit.AuditMessage;
+import org.apache.hadoop.ozone.audit.Auditor;
+import org.apache.hadoop.ozone.audit.SCMAction;
import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
@@ -91,6 +100,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
@@ -106,11 +116,14 @@ import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
* Protocol Handler for Datanode Protocol.
*/
public class SCMDatanodeProtocolServer implements
- StorageContainerDatanodeProtocol {
+ StorageContainerDatanodeProtocol, Auditor {
private static final Logger LOG = LoggerFactory.getLogger(
SCMDatanodeProtocolServer.class);
+ private static final AuditLogger AUDIT =
+ new AuditLogger(AuditLoggerType.SCMLOGGER);
+
/**
* The RPC server that listens to requests from DataNodes.
*/
@@ -179,8 +192,21 @@ public class SCMDatanodeProtocolServer implements
public SCMVersionResponseProto getVersion(SCMVersionRequestProto
versionRequest)
throws IOException {
- return scm.getScmNodeManager().getVersion(versionRequest)
- .getProtobufMessage();
+ boolean auditSuccess = true;
+ try {
+ return scm.getScmNodeManager().getVersion(versionRequest)
+ .getProtobufMessage();
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logReadFailure(
+ buildAuditMessageForFailure(SCMAction.GET_VERSION, null, ex));
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logReadSuccess(
+ buildAuditMessageForSuccess(SCMAction.GET_VERSION, null));
+ }
+ }
}
@Override
@@ -192,6 +218,10 @@ public class SCMDatanodeProtocolServer implements
throws IOException {
DatanodeDetails datanodeDetails = DatanodeDetails
.getFromProtoBuf(datanodeDetailsProto);
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("datanodeDetails", datanodeDetails.toString());
+
// TODO : Return the list of Nodes that forms the SCM HA.
RegisteredCommand registeredCommand = scm.getScmNodeManager()
.register(datanodeDetails, nodeReport, pipelineReportsProto);
@@ -207,7 +237,19 @@ public class SCMDatanodeProtocolServer implements
new PipelineReportFromDatanode(datanodeDetails,
pipelineReportsProto));
}
- return getRegisteredResponse(registeredCommand);
+ try {
+ return getRegisteredResponse(registeredCommand);
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logWriteFailure(
+ buildAuditMessageForFailure(SCMAction.REGISTER, auditMap, ex));
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.REGISTER, auditMap));
+ }
+ }
}
@VisibleForTesting
@@ -229,9 +271,27 @@ public class SCMDatanodeProtocolServer implements
for (SCMCommand cmd : heartbeatDispatcher.dispatch(heartbeat)) {
cmdResponses.add(getCommandResponse(cmd));
}
- return SCMHeartbeatResponseProto.newBuilder()
- .setDatanodeUUID(heartbeat.getDatanodeDetails().getUuid())
- .addAllCommands(cmdResponses).build();
+ boolean auditSuccess = true;
+ Map<String, String> auditMap = Maps.newHashMap();
+ auditMap.put("datanodeUUID", heartbeat.getDatanodeDetails().getUuid());
+ auditMap.put("command", flatten(cmdResponses.toString()));
+ try {
+ return SCMHeartbeatResponseProto.newBuilder()
+ .setDatanodeUUID(heartbeat.getDatanodeDetails().getUuid())
+ .addAllCommands(cmdResponses).build();
+ } catch (Exception ex) {
+ auditSuccess = false;
+ AUDIT.logWriteFailure(
+ buildAuditMessageForFailure(SCMAction.SEND_HEARTBEAT, auditMap, ex)
+ );
+ throw ex;
+ } finally {
+ if(auditSuccess) {
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.SEND_HEARTBEAT, auditMap)
+ );
+ }
+ }
}
/**
@@ -302,6 +362,43 @@ public class SCMDatanodeProtocolServer implements
IOUtils.cleanupWithLogger(LOG, scm.getScmNodeManager());
}
+ @Override
+ public AuditMessage buildAuditMessageForSuccess(
+ AuditAction op, Map<String, String> auditMap) {
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.SUCCESS.toString())
+ .withException(null)
+ .build();
+ }
+
+ @Override
+ public AuditMessage buildAuditMessageForFailure(AuditAction op, Map<String,
+ String> auditMap, Throwable throwable) {
+ return new AuditMessage.Builder()
+ .setUser((Server.getRemoteUser() == null) ? null :
+ Server.getRemoteUser().getUserName())
+ .atIp((Server.getRemoteIp() == null) ? null :
+ Server.getRemoteIp().getHostAddress())
+ .forOperation(op.getAction())
+ .withParams(auditMap)
+ .withResult(AuditEventStatus.FAILURE.toString())
+ .withException(throwable)
+ .build();
+ }
+
+ private static String flatten(String input) {
+ return input
+ .replaceAll(System.lineSeparator(), " ")
+ .trim()
+ .replaceAll(" +", " ");
+ }
+
/**
* Wrapper class for events with the datanode origin.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index e034082..bea01f0 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -115,6 +115,7 @@ function ozonecmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManager'
hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS"
+ HDFS_STORAGECONTAINERMANAGER_OPTS="${HDFS_STORAGECONTAINERMANAGER_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/scm-audit-log4j2.properties"
HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}"
OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm"
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index 55927a7..66181fd 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -87,6 +87,7 @@ run mkdir -p ./libexec
run cp -r "${ROOT}/hadoop-common-project/hadoop-common/src/main/conf" "etc/hadoop"
run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/om-audit-log4j2.properties" "etc/hadoop"
run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/dn-audit-log4j2.properties" "etc/hadoop"
+run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/scm-audit-log4j2.properties" "etc/hadoop"
run cp "${ROOT}/hadoop-ozone/dist/src/main/conf/ozone-site.xml" "etc/hadoop"
run cp -f "${ROOT}/hadoop-ozone/dist/src/main/conf/log4j.properties" "etc/hadoop"
run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop" "bin/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b368f2/hadoop-ozone/dist/src/main/conf/scm-audit-log4j2.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/conf/scm-audit-log4j2.properties b/hadoop-ozone/dist/src/main/conf/scm-audit-log4j2.properties
new file mode 100644
index 0000000..3f81561
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/conf/scm-audit-log4j2.properties
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# <p>
+# http://www.apache.org/licenses/LICENSE-2.0
+# <p>
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+name=PropertiesConfig
+
+# Checks for config change periodically and reloads
+monitorInterval=30
+
+filter=read,write
+# filter.read.onMatch=DENY avoids logging all READ events
+# filter.read.onMatch=ACCEPT permits logging all READ events
+# The above two settings ignore the log levels in configuration
+# filter.read.onMatch=NEUTRAL permits logging of only those READ events
+# which are attempted at log level equal or greater than log level specified
+# in the configuration
+filter.read.type=MarkerFilter
+filter.read.marker=READ
+filter.read.onMatch=DENY
+filter.read.onMismatch=NEUTRAL
+
+# filter.write.onMatch=DENY avoids logging all WRITE events
+# filter.write.onMatch=ACCEPT permits logging all WRITE events
+# The above two settings ignore the log levels in configuration
+# filter.write.onMatch=NEUTRAL permits logging of only those WRITE events
+# which are attempted at log level equal or greater than log level specified
+# in the configuration
+filter.write.type=MarkerFilter
+filter.write.marker=WRITE
+filter.write.onMatch=NEUTRAL
+filter.write.onMismatch=NEUTRAL
+
+# Log Levels are organized from most specific to least:
+# OFF (most specific, no logging)
+# FATAL (most specific, little data)
+# ERROR
+# WARN
+# INFO
+# DEBUG
+# TRACE (least specific, a lot of data)
+# ALL (least specific, all data)
+
+# Uncomment following section to enable logging to console appender also
+#appenders=console, rolling
+#appender.console.type=Console
+#appender.console.name=STDOUT
+#appender.console.layout.type=PatternLayout
+#appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+
+# Comment this line when using both console and rolling appenders
+appenders=rolling
+
+#Rolling File Appender with size & time thresholds.
+#Rolling is triggered when either threshold is breached.
+#The rolled over file is compressed by default
+#Time interval is specified in seconds 86400s=1 day
+appender.rolling.type=RollingFile
+appender.rolling.name=RollingFile
+appender.rolling.fileName =${sys:hadoop.log.dir}/scm-audit-${hostName}.log
+appender.rolling.filePattern=${sys:hadoop.log.dir}/scm-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+appender.rolling.layout.type=PatternLayout
+appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+appender.rolling.policies.type=Policies
+appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval=86400
+appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size=64MB
+
+loggers=audit
+logger.audit.type=AsyncLogger
+logger.audit.name=SCMAudit
+logger.audit.level=INFO
+logger.audit.appenderRefs=rolling
+logger.audit.appenderRef.file.ref=RollingFile
+
+rootLogger.level=INFO
+#rootLogger.appenderRefs=stdout
+#rootLogger.appenderRef.stdout.ref=STDOUT
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/22] hadoop git commit: HADOOP-16004. ABFS: Convert 404 error
response in AbfsInputStream and AbfsOutPutStream to FileNotFoundException.
Posted by sh...@apache.org.
HADOOP-16004. ABFS: Convert 404 error response in AbfsInputStream and AbfsOutPutStream to FileNotFoundException.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/346c0c8a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/346c0c8a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/346c0c8a
Branch: refs/heads/HDFS-12943
Commit: 346c0c8aff0b206d45f34dbce4fcc81364115d95
Parents: 62df607
Author: Da Zhou <da...@microsoft.com>
Authored: Mon Dec 17 11:15:20 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Dec 17 11:15:20 2018 +0000
----------------------------------------------------------------------
.../fs/azurebfs/services/AbfsInputStream.java | 9 ++++
.../fs/azurebfs/services/AbfsOutputStream.java | 16 +++++-
.../azurebfs/ITestAzureBlobFileSystemE2E.java | 51 ++++++++++++++++++--
3 files changed, 72 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/346c0c8a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
index 960579d..9593679 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
@@ -19,13 +19,16 @@
package org.apache.hadoop.fs.azurebfs.services;
import java.io.EOFException;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.HttpURLConnection;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
/**
@@ -225,6 +228,12 @@ public class AbfsInputStream extends FSInputStream {
try {
op = client.read(path, position, b, offset, length, tolerateOobAppends ? "*" : eTag);
} catch (AzureBlobFileSystemException ex) {
+ if (ex instanceof AbfsRestOperationException) {
+ AbfsRestOperationException ere = (AbfsRestOperationException) ex;
+ if (ere.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ throw new FileNotFoundException(ere.getMessage());
+ }
+ }
throw new IOException(ex);
}
long bytesRead = op.getResult().getBytesReceived();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/346c0c8a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
index 7e43090..85db774 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.fs.azurebfs.services;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.OutputStream;
+import java.net.HttpURLConnection;
import java.util.Locale;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.LinkedBlockingQueue;
@@ -33,10 +35,11 @@ import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.Syncable;
-import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
/**
* The BlobFsOutputStream for Rest AbfsClient.
@@ -290,6 +293,12 @@ public class AbfsOutputStream extends OutputStream implements Syncable, StreamCa
try {
writeOperation.task.get();
} catch (Exception ex) {
+ if (ex.getCause() instanceof AbfsRestOperationException) {
+ if (((AbfsRestOperationException) ex.getCause()).getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ throw new FileNotFoundException(ex.getMessage());
+ }
+ }
+
if (ex.getCause() instanceof AzureBlobFileSystemException) {
ex = (AzureBlobFileSystemException) ex.getCause();
}
@@ -313,6 +322,11 @@ public class AbfsOutputStream extends OutputStream implements Syncable, StreamCa
try {
client.flush(path, offset, retainUncommitedData);
} catch (AzureBlobFileSystemException ex) {
+ if (ex instanceof AbfsRestOperationException) {
+ if (((AbfsRestOperationException) ex).getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+ throw new FileNotFoundException(ex.getMessage());
+ }
+ }
throw new IOException(ex);
}
this.lastFlushOffset = offset;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/346c0c8a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
index 2e99491..9e22790 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.azurebfs;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
@@ -30,9 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertArrayEquals;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test end to end between ABFS client and ABFS server.
@@ -136,6 +135,52 @@ public class ITestAzureBlobFileSystemE2E extends AbstractAbfsIntegrationTest {
inputStream.close();
}
+ @Test
+ public void testReadWithFileNotFoundException() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path testFilePath = new Path(methodName.getMethodName());
+ testWriteOneByteToFile(testFilePath);
+
+ FSDataInputStream inputStream = fs.open(testFilePath, TEST_DEFAULT_BUFFER_SIZE);
+ fs.delete(testFilePath, true);
+ assertFalse(fs.exists(testFilePath));
+
+ intercept(FileNotFoundException.class,
+ () -> inputStream.read(new byte[1]));
+ }
+
+ @Test
+ public void testWriteWithFileNotFoundException() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path testFilePath = new Path(methodName.getMethodName());
+
+ FSDataOutputStream stream = fs.create(testFilePath);
+ assertTrue(fs.exists(testFilePath));
+ stream.write(TEST_BYTE);
+
+ fs.delete(testFilePath, true);
+ assertFalse(fs.exists(testFilePath));
+
+ // trigger append call
+ intercept(FileNotFoundException.class,
+ () -> stream.close());
+ }
+
+ @Test
+ public void testFlushWithFileNotFoundException() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path testFilePath = new Path(methodName.getMethodName());
+
+ FSDataOutputStream stream = fs.create(testFilePath);
+ assertTrue(fs.exists(testFilePath));
+
+ fs.delete(testFilePath, true);
+ assertFalse(fs.exists(testFilePath));
+
+ intercept(FileNotFoundException.class,
+ () -> stream.close());
+ }
+
private void testWriteOneByteToFile(Path testFilePath) throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
try(FSDataOutputStream stream = fs.create(testFilePath)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/22] hadoop git commit: HDFS-14006. Refactor name node to allow
different token verification implementations. Contributed by CR Hota.
Posted by sh...@apache.org.
HDFS-14006. Refactor name node to allow different token verification implementations. Contributed by CR Hota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00d5e631
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00d5e631
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00d5e631
Branch: refs/heads/HDFS-12943
Commit: 00d5e631b596f8712600879366e5283829e7ee5d
Parents: f858f18
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Dec 14 11:10:54 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Dec 14 11:10:54 2018 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/common/JspHelper.java | 8 ++---
.../hdfs/server/common/TokenVerifier.java | 35 ++++++++++++++++++++
.../hadoop/hdfs/server/namenode/NameNode.java | 9 ++++-
.../server/namenode/NameNodeHttpServer.java | 6 ++++
4 files changed, 53 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00d5e631/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
index 498a093..eb488e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
@@ -23,7 +23,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
@@ -176,10 +175,11 @@ public class JspHelper {
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.readFields(in);
if (context != null) {
- final NameNode nn = NameNodeHttpServer.getNameNodeFromContext(context);
- if (nn != null) {
+ final TokenVerifier<DelegationTokenIdentifier> tokenVerifier =
+ NameNodeHttpServer.getTokenVerifierFromContext(context);
+ if (tokenVerifier != null) {
// Verify the token.
- nn.getNamesystem().verifyToken(id, token.getPassword());
+ tokenVerifier.verifyToken(id, token.getPassword());
}
}
UserGroupInformation ugi = id.getUser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00d5e631/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/TokenVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/TokenVerifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/TokenVerifier.java
new file mode 100644
index 0000000..5691f0c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/TokenVerifier.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.common;
+
+import java.io.IOException;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+
+/**
+ * Interface to verify delegation tokens passed through WebHDFS.
+ * Implementations are intercepted by JspHelper that pass delegation token
+ * for verification.
+ */
+public interface TokenVerifier<T extends AbstractDelegationTokenIdentifier> {
+
+ /* Verify delegation token passed through WebHDFS
+ * Name node, Router implement this for JspHelper to verify token
+ */
+ void verifyToken(T t, byte[] password) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00d5e631/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 471abac..37b694b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap;
import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -55,6 +56,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeSt
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.common.TokenVerifier;
import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -208,7 +210,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DE
**********************************************************/
@InterfaceAudience.Private
public class NameNode extends ReconfigurableBase implements
- NameNodeStatusMXBean {
+ NameNodeStatusMXBean, TokenVerifier<DelegationTokenIdentifier> {
static{
HdfsConfiguration.init();
}
@@ -656,6 +658,11 @@ public class NameNode extends ReconfigurableBase implements
return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
}
+ @Override
+ public void verifyToken(DelegationTokenIdentifier id, byte[] password)
+ throws IOException {
+ namesystem.verifyToken(id, password);
+ }
/**
* Login as the configured user for the NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00d5e631/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
index 4d27bd9..69dadf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap;
import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.common.TokenVerifier;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.web.AuthFilter;
@@ -319,6 +320,11 @@ public class NameNodeHttpServer {
return (NameNode)context.getAttribute(NAMENODE_ATTRIBUTE_KEY);
}
+ public static TokenVerifier
+ getTokenVerifierFromContext(ServletContext context) {
+ return (TokenVerifier) context.getAttribute(NAMENODE_ATTRIBUTE_KEY);
+ }
+
static Configuration getConfFromContext(ServletContext context) {
return (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/22] hadoop git commit: HADOOP-16000. Remove TLSv1 and SSLv2Hello
from the default value of hadoop.ssl.enabled.protocols
Posted by sh...@apache.org.
HADOOP-16000. Remove TLSv1 and SSLv2Hello from the default value of hadoop.ssl.enabled.protocols
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ea29b73
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ea29b73
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ea29b73
Branch: refs/heads/HDFS-12943
Commit: 1ea29b7385bc1545d7aa452549073d81f0c24b36
Parents: 26e55d4
Author: Gabor Bota <ga...@cloudera.com>
Authored: Wed Dec 12 07:28:35 2018 +0100
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Sat Dec 15 10:53:52 2018 +0900
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/security/ssl/SSLFactory.java | 2 +-
.../hadoop-common/src/main/resources/core-default.xml | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea29b73/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index 10c1d7d..3189b44 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -72,7 +72,7 @@ public class SSLFactory implements ConnectionConfigurator {
public static final String SSL_ENABLED_PROTOCOLS_KEY =
"hadoop.ssl.enabled.protocols";
public static final String SSL_ENABLED_PROTOCOLS_DEFAULT =
- "TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2";
+ "TLSv1.1,TLSv1.2";
public static final String SSL_SERVER_NEED_CLIENT_AUTH =
"ssl.server.need.client.auth";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea29b73/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f7d61c6..ddcee2f 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2417,9 +2417,10 @@
<property>
<name>hadoop.ssl.enabled.protocols</name>
- <value>TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2</value>
+ <value>TLSv1.1,TLSv1.2</value>
<description>
- The supported SSL protocols.
+ The supported SSL protocols. The parameter will only used from
+ DatanodeHttpServer.
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/22] hadoop git commit: HDFS-14138. [SBN read] Description errors
in the comparison logic of transaction ID. Contributed by xiangheng.
Posted by sh...@apache.org.
HDFS-14138. [SBN read] Description errors in the comparison logic of transaction ID. Contributed by xiangheng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b10b80a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b10b80a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b10b80a
Branch: refs/heads/HDFS-12943
Commit: 5b10b80aee1d5798daa82b89c7d9638ed865a7d8
Parents: 9a9e366
Author: Chen Liang <cl...@apache.org>
Authored: Fri Dec 14 13:06:38 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Dec 17 16:01:21 2018 -0800
----------------------------------------------------------------------
.../hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b10b80a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index bc1a62d..bab2893 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -2784,7 +2784,7 @@ public abstract class Server {
call.getClientStateId() > alignmentContext.getLastSeenStateId()) {
/*
* The call processing should be postponed until the client call's
- * state id is aligned (>=) with the server state id.
+ * state id is aligned (<=) with the server state id.
* NOTE:
* Inserting the call back to the queue can change the order of call
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/22] hadoop git commit: YARN-9122. Add table of contents to YARN
Service API document. Contributed by Zhankun Tang.
Posted by sh...@apache.org.
YARN-9122. Add table of contents to YARN Service API document. Contributed by Zhankun Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26e55d4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26e55d4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26e55d4c
Branch: refs/heads/HDFS-12943
Commit: 26e55d4ceb691f47801fa5f0ba1147a5bc413f96
Parents: b2d7204
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sat Dec 15 10:29:47 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Sat Dec 15 10:29:47 2018 +0900
----------------------------------------------------------------------
.../markdown/yarn-service/YarnServiceAPI.md | 57 +++++++++++++++-----
1 file changed, 43 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26e55d4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index e863ccc..5228eab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -12,8 +12,11 @@
limitations under the License. See accompanying LICENSE file.
-->
-# YARN Service API
+# YARN Service API
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
+
+## Introduction
Bringing a new service on YARN today is not a simple experience. The APIs of existing
frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs)
or writing a complex spec (for declarative frameworks).
@@ -433,9 +436,11 @@ The current status of a submitted service, returned as a response to the GET API
## Examples
### Create a simple single-component service with most attribute values as defaults
+```
POST URL - http://localhost:8088/app/v1/services
+```
-##### POST Request JSON
+#### POST Request JSON
```json
{
"name": "hello-world",
@@ -460,8 +465,10 @@ POST URL - http://localhost:8088/app/v1/services
}
```
-##### GET Response JSON
+#### GET Response JSON
+```
GET URL - http://localhost:8088/app/v1/services/hello-world
+```
Note, lifetime value of -1 means unlimited lifetime.
@@ -522,9 +529,11 @@ Note, lifetime value of -1 means unlimited lifetime.
```
### Update to modify the lifetime of a service
+```
PUT URL - http://localhost:8088/app/v1/services/hello-world
+```
-##### PUT Request JSON
+#### PUT Request JSON
Note, irrespective of what the current lifetime value is, this update request will set the lifetime of the service to be 3600 seconds (1 hour) from the time the request is submitted. Hence, if a a service has remaining lifetime of 5 mins (say) and would like to extend it to an hour OR if an application has remaining lifetime of 5 hours (say) and would like to reduce it down to an hour, then for both scenarios you need to submit the same request below.
@@ -534,9 +543,11 @@ Note, irrespective of what the current lifetime value is, this update request wi
}
```
### Stop a service
+```
PUT URL - http://localhost:8088/app/v1/services/hello-world
+```
-##### PUT Request JSON
+#### PUT Request JSON
```json
{
"state": "STOPPED"
@@ -544,9 +555,11 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world
```
### Start a service
+```
PUT URL - http://localhost:8088/app/v1/services/hello-world
+```
-##### PUT Request JSON
+#### PUT Request JSON
```json
{
"state": "STARTED"
@@ -554,9 +567,11 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world
```
### Update to flex up/down the number of containers (instances) of a component of a service
+```
PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello
+```
-##### PUT Request JSON
+#### PUT Request JSON
```json
{
"number_of_containers": 3
@@ -564,9 +579,11 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world/components/hello
```
Alternatively, you can specify the entire "components" section instead.
-
+```
PUT URL - http://localhost:8088/app/v1/services/hello-world
-##### PUT Request JSON
+```
+
+#### PUT Request JSON
```json
{
"state": "FLEX",
@@ -581,14 +598,18 @@ PUT URL - http://localhost:8088/app/v1/services/hello-world
```
### Destroy a service
+```
DELETE URL - http://localhost:8088/app/v1/services/hello-world
+```
***
### Create a complicated service - HBase
+```
POST URL - http://localhost:8088:/app/v1/services/hbase-app-1
+```
-##### POST Request JSON
+#### POST Request JSON
```json
{
@@ -685,9 +706,11 @@ POST URL - http://localhost:8088:/app/v1/services/hbase-app-1
```
### Create a service requesting GPUs in addition to CPUs and RAM
+```
POST URL - http://localhost:8088/app/v1/services
+```
-##### POST Request JSON
+#### POST Request JSON
```json
{
"name": "hello-world",
@@ -719,9 +742,11 @@ POST URL - http://localhost:8088/app/v1/services
```
### Create a service with a component requesting anti-affinity placement policy
+```
POST URL - http://localhost:8088/app/v1/services
+```
-##### POST Request JSON
+#### POST Request JSON
```json
{
"name": "hello-world",
@@ -765,8 +790,10 @@ POST URL - http://localhost:8088/app/v1/services
}
```
-##### GET Response JSON
+#### GET Response JSON
+```
GET URL - http://localhost:8088/app/v1/services/hello-world
+```
Note, for an anti-affinity component no more than 1 container will be allocated
in a specific node. In this example, 3 containers have been requested by
@@ -861,9 +888,11 @@ non-STABLE state.
```
### Create a service with health threshold monitor enabled for a component
+```
POST URL - http://localhost:8088/app/v1/services
+```
-##### POST Request JSON
+#### POST Request JSON
```json
{
"name": "hello-world",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/22] hadoop git commit: YARN-9125. Fixed Carriage Return detection
in Docker container launch command. Contributed by Billie Rinaldi
Posted by sh...@apache.org.
YARN-9125. Fixed Carriage Return detection in Docker container launch command.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2d7204e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2d7204e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2d7204e
Branch: refs/heads/HDFS-12943
Commit: b2d7204ed0b64405a0adbe07e3eaf385e046efa1
Parents: ca90b24
Author: Eric Yang <ey...@apache.org>
Authored: Fri Dec 14 17:52:26 2018 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Dec 14 17:52:26 2018 -0500
----------------------------------------------------------------------
.../linux/runtime/docker/DockerClient.java | 8 ++--
.../linux/runtime/docker/TestDockerClient.java | 46 ++++++++++++++++++++
2 files changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d7204e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
index 3a516c4..6cad26e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -105,13 +105,13 @@ public final class DockerClient {
"'=' found in entry for docker command file, key = " + entry
.getKey() + "; value = " + entry.getValue());
}
- if (entry.getValue().contains("\n")) {
+ String value = StringUtils.join(",", entry.getValue());
+ if (value.contains("\n")) {
throw new ContainerExecutionException(
"'\\n' found in entry for docker command file, key = " + entry
- .getKey() + "; value = " + entry.getValue());
+ .getKey() + "; value = " + value);
}
- printWriter.println(" " + entry.getKey() + "=" + StringUtils
- .join(",", entry.getValue()));
+ printWriter.println(" " + entry.getKey() + "=" + value);
}
if (cmd instanceof DockerRunCommand) {
DockerRunCommand runCommand = (DockerRunCommand) cmd;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d7204e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerClient.java
index 31645bc..bd01c7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerClient.java
@@ -27,13 +27,16 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
+import java.util.Arrays;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
@@ -77,4 +80,47 @@ public class TestDockerClient {
File tmpFile = new File(tmpPath);
assertTrue(tmpFile + " was not created", tmpFile.exists());
}
+
+ @Test
+ public void testCommandValidation() throws Exception {
+ String absRoot = TEST_ROOT_DIR.getAbsolutePath();
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId cid = ContainerId.newContainerId(attemptId, 1);
+ Configuration conf = new Configuration();
+ conf.set("hadoop.tmp.dir", absRoot);
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, absRoot);
+ conf.set(YarnConfiguration.NM_LOG_DIRS, absRoot);
+ LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
+ Context mockContext = mock(Context.class);
+ doReturn(conf).when(mockContext).getConf();
+ doReturn(dirsHandler).when(mockContext).getLocalDirsHandler();
+
+ DockerClient dockerClient = new DockerClient();
+ dirsHandler.init(conf);
+ dirsHandler.start();
+ try {
+ DockerRunCommand dockerCmd = new DockerRunCommand(cid.toString(), "user",
+ "image");
+ dockerCmd.addCommandArguments("prop=bad", "val");
+ dockerClient.writeCommandToTempFile(dockerCmd, cid,
+ mockContext);
+ fail("Expected exception writing command file");
+ } catch (ContainerExecutionException e) {
+ assertTrue("Expected key validation error", e.getMessage()
+ .contains("'=' found in entry for docker command file"));
+ }
+
+ try {
+ DockerRunCommand dockerCmd = new DockerRunCommand(cid.toString(), "user",
+ "image");
+ dockerCmd.setOverrideCommandWithArgs(Arrays.asList("sleep", "1000\n"));
+ dockerClient.writeCommandToTempFile(dockerCmd, cid, mockContext);
+ fail("Expected exception writing command file");
+ } catch (ContainerExecutionException e) {
+ assertTrue("Expected value validation error", e.getMessage()
+ .contains("'\\n' found in entry for docker command file"));
+ }
+ dirsHandler.stop();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/22] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Posted by sh...@apache.org.
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d891a54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d891a54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d891a54
Branch: refs/heads/HDFS-12943
Commit: 4d891a54101b2efb73d76b6c9e75c4140597832e
Parents: f19f4cb 94b368f
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Mon Dec 17 16:02:36 2018 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Dec 17 16:02:36 2018 -0800
----------------------------------------------------------------------
LICENSE.txt | 33 +
NOTICE.txt | 7 +
.../hadoop-client-minicluster/pom.xml | 8 +
.../apache/hadoop/fs/AbstractFileSystem.java | 10 +
.../java/org/apache/hadoop/fs/FileContext.java | 18 +
.../java/org/apache/hadoop/fs/FileSystem.java | 10 +
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +
.../java/org/apache/hadoop/fs/FilterFs.java | 5 +
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 +
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 5 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 13 +
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +
.../hadoop/security/LdapGroupsMapping.java | 23 +-
.../apache/hadoop/security/ssl/SSLFactory.java | 2 +-
.../hadoop/security/token/DtFileOperations.java | 25 +-
.../org/apache/hadoop/security/token/Token.java | 44 +-
.../src/main/resources/core-default.xml | 15 +-
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +
.../fs/viewfs/ViewFileSystemBaseTest.java | 5 +
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 4 +-
.../hadoop/security/TestLdapGroupsMapping.java | 42 +-
.../apache/hadoop/security/token/TestToken.java | 22 +-
.../hadoop/hdds/scm/XceiverClientGrpc.java | 6 +-
.../hadoop/hdds/scm/XceiverClientRatis.java | 110 +-
.../hdds/scm/storage/ChunkOutputStream.java | 426 +-
.../hdds/scm/XceiverClientAsyncReply.java | 44 +-
.../hadoop/hdds/scm/XceiverClientSpi.java | 19 +-
.../scm/storage/ContainerProtocolCalls.java | 1 -
.../apache/hadoop/ozone/OzoneConfigKeys.java | 8 +
.../org/apache/hadoop/ozone/OzoneConsts.java | 13 +
.../apache/hadoop/ozone/audit/SCMAction.java | 45 +
.../apache/hadoop/ozone/common/BlockGroup.java | 8 +
.../java/org/apache/hadoop/utils/db/Codec.java | 3 +
.../apache/hadoop/utils/db/CodecRegistry.java | 19 +
.../org/apache/hadoop/utils/db/DBStore.java | 13 +-
.../apache/hadoop/utils/db/DBStoreBuilder.java | 9 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 106 +-
.../org/apache/hadoop/utils/db/TypedTable.java | 3 +
.../common/src/main/resources/ozone-default.xml | 123 +-
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 15 +-
.../server/ratis/DispatcherContext.java | 4 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 106 +-
.../scm/server/SCMClientProtocolServer.java | 227 +-
.../scm/server/SCMDatanodeProtocolServer.java | 111 +-
.../main/java/org/apache/hadoop/fs/Hdfs.java | 5 +
.../hadoop/hdfs/DistributedFileSystem.java | 6 +-
.../datanode/ReplicaNotFoundException.java | 8 +-
.../ha/RequestHedgingProxyProvider.java | 209 +-
.../hdfs/util/CombinedHostsFileReader.java | 41 +-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 11 +
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 33 +
.../hadoop/hdfs/web/resources/GetOpParam.java | 2 +
.../hadoop/hdfs/web/resources/PostOpParam.java | 2 +
.../hadoop/hdfs/web/resources/PutOpParam.java | 1 +
.../ha/TestRequestHedgingProxyProvider.java | 60 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../hadoop/hdfs/protocol/CacheDirective.java | 3 +-
.../hdfs/server/aliasmap/InMemoryAliasMap.java | 216 +-
.../aliasmap/InMemoryLevelDBAliasMapServer.java | 8 +
.../hadoop/hdfs/server/common/JspHelper.java | 8 +-
.../hdfs/server/common/TokenVerifier.java | 35 +
.../hdfs/server/namenode/CacheManager.java | 38 +-
.../namenode/ErasureCodingPolicyManager.java | 13 +
.../hdfs/server/namenode/ImageServlet.java | 32 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 20 +-
.../server/namenode/NameNodeHttpServer.java | 21 +
.../hdfs/server/namenode/TransferFsImage.java | 21 +-
.../server/namenode/ha/BootstrapStandby.java | 91 +-
.../namenode/metrics/NameNodeMetrics.java | 6 +
.../web/resources/NamenodeWebHdfsMethods.java | 15 +-
.../src/main/resources/hdfs-default.xml | 8 +
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 89 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 23 +
.../hadoop/hdfs/TestErasureCodingAddConfig.java | 79 +
.../hadoop/hdfs/TestMaintenanceState.java | 2 +
.../java/org/apache/hadoop/hdfs/TestPread.java | 3 +-
.../server/aliasmap/ITestInMemoryAliasMap.java | 30 +
.../server/aliasmap/TestInMemoryAliasMap.java | 45 -
.../impl/TestInMemoryLevelDBAliasMapClient.java | 9 -
.../namenode/TestListCorruptFileBlocks.java | 24 +-
.../namenode/ha/TestBootstrapAliasmap.java | 99 +
.../TestBlockStorageMovementAttemptedItems.java | 1 -
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 18 +-
.../v2/app/job/impl/TestTaskAttempt.java | 38 +-
.../PluggableShuffleAndPluggableSort.md | 4 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 44 +-
.../hadoop-mapreduce-client-uploader/pom.xml | 6 +
.../mapred/uploader/FrameworkUploader.java | 52 +
.../mapred/uploader/TestFrameworkUploader.java | 58 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 37 +
.../ozone/client/io/ChunkGroupOutputStream.java | 166 +-
.../ozone/client/io/OzoneOutputStream.java | 10 +
.../ozone/client/protocol/ClientProtocol.java | 22 +
.../hadoop/ozone/client/rest/RestClient.java | 24 +
.../hadoop/ozone/client/rpc/RpcClient.java | 73 +
hadoop-ozone/common/src/main/bin/ozone | 1 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 4 +-
.../apache/hadoop/ozone/om/OMConfigKeys.java | 60 +
.../hadoop/ozone/om/OMMetadataManager.java | 58 +-
.../ozone/om/codec/OmBucketInfoCodec.java | 51 +
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 51 +
.../ozone/om/codec/OmMultipartKeyInfoCodec.java | 56 +
.../ozone/om/codec/OmVolumeArgsCodec.java | 51 +
.../hadoop/ozone/om/codec/VolumeListCodec.java | 50 +
.../hadoop/ozone/om/codec/package-info.java | 24 +
.../hadoop/ozone/om/helpers/OmKeyArgs.java | 42 +-
.../hadoop/ozone/om/helpers/OmKeyInfo.java | 30 +-
.../OmMultipartCommitUploadPartInfo.java | 34 +
.../ozone/om/helpers/OmMultipartInfo.java | 77 +
.../ozone/om/helpers/OmMultipartKeyInfo.java | 105 +
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 3 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 17 +-
...neManagerProtocolClientSideTranslatorPB.java | 95 +-
.../ozone/security/acl/IAccessAuthorizer.java | 153 +
.../hadoop/ozone/security/acl/IOzoneObj.java | 24 +
.../security/acl/OzoneAccessAuthorizer.java | 29 +
.../ozone/security/acl/OzoneAclException.java | 71 +
.../hadoop/ozone/security/acl/OzoneObj.java | 109 +
.../hadoop/ozone/security/acl/OzoneObjInfo.java | 118 +
.../ozone/security/acl/RequestContext.java | 122 +
.../hadoop/ozone/security/acl/package-info.java | 22 +
.../src/main/proto/OzoneManagerProtocol.proto | 48 +
.../om/codec/TestOmMultipartKeyInfoCodec.java | 54 +
.../hadoop/ozone/om/codec/package-info.java | 24 +
.../ozone/security/acl/TestOzoneObjInfo.java | 87 +
hadoop-ozone/dist/Dockerfile | 21 +
hadoop-ozone/dist/README.md | 85 +
.../dist/dev-support/bin/dist-layout-stitching | 9 +
hadoop-ozone/dist/skaffold.yaml | 25 +
hadoop-ozone/dist/src/main/blockade/README.md | 26 +
.../src/main/blockade/blockadeUtils/__init__.py | 14 +
.../src/main/blockade/blockadeUtils/blockade.py | 59 +
.../src/main/blockade/clusterUtils/__init__.py | 14 +
.../main/blockade/clusterUtils/cluster_utils.py | 75 +
hadoop-ozone/dist/src/main/blockade/conftest.py | 65 +
.../dist/src/main/blockade/test_blockade.py | 54 +
.../src/main/conf/scm-audit-log4j2.properties | 90 +
.../src/main/k8s/ozone/config-configmap.yaml | 34 +
.../main/k8s/ozone/datanode-public-service.yaml | 29 +
.../src/main/k8s/ozone/datanode-service.yaml | 29 +
.../main/k8s/ozone/datanode-statefulset.yaml | 52 +
.../src/main/k8s/ozone/om-public-service.yaml | 29 +
.../dist/src/main/k8s/ozone/om-service.yaml | 29 +
.../dist/src/main/k8s/ozone/om-statefulset.yaml | 72 +
.../src/main/k8s/ozone/s3g-public-service.yaml | 29 +
.../dist/src/main/k8s/ozone/s3g-service.yaml | 29 +
.../src/main/k8s/ozone/s3g-statefulset.yaml | 49 +
.../src/main/k8s/ozone/scm-public-service.yaml | 29 +
.../dist/src/main/k8s/ozone/scm-service.yaml | 29 +
.../src/main/k8s/ozone/scm-statefulset.yaml | 67 +
.../dist/src/main/k8s/prometheus/configmap.yaml | 51 +
.../src/main/k8s/prometheus/deployment.yaml | 46 +
.../prometheus/prometheus-public-serivce.yaml | 28 +
.../dist/src/main/k8s/prometheus/role.yaml | 31 +
.../src/main/k8s/prometheus/rolebindng.yaml | 27 +
.../main/k8s/prometheus/scm-public-service.yaml | 29 +
.../main/k8s/prometheus/service-account.yaml | 19 +
.../dist/src/main/k8s/prometheus/service.yaml | 25 +
.../rpc/TestCloseContainerHandlingByClient.java | 19 +-
.../client/rpc/TestFailureHandlingByClient.java | 56 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 256 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 176 +
.../apache/hadoop/ozone/om/TestOmMetrics.java | 18 +-
.../hadoop/ozone/om/TestOzoneManager.java | 32 +-
.../hadoop/ozone/om/BucketManagerImpl.java | 37 +-
.../hadoop/ozone/om/KeyDeletingService.java | 3 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 15 +
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 345 +-
.../org/apache/hadoop/ozone/om/OMMetrics.java | 30 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 212 +-
.../apache/hadoop/ozone/om/OzoneManager.java | 275 +-
.../hadoop/ozone/om/VolumeManagerImpl.java | 109 +-
.../hadoop/ozone/om/exceptions/OMException.java | 5 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 271 +
.../hadoop/ozone/om/ratis/package-info.java | 22 +
...neManagerProtocolServerSideTranslatorPB.java | 72 +
.../hadoop/ozone/om/TestBucketManagerImpl.java | 50 +-
.../hadoop/ozone/om/TestChunkStreams.java | 78 -
.../hadoop/ozone/om/TestKeyDeletingService.java | 41 +-
.../hadoop/ozone/om/TestKeyManagerImpl.java | 72 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 69 +
.../ozone/s3/header/AuthorizationHeaderV4.java | 28 +-
.../s3/header/TestAuthorizationHeaderV4.java | 60 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 16 +-
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 44 +-
.../site/markdown/tools/hadoop-aws/s3guard.md | 2 +-
.../site/markdown/tools/hadoop-aws/testing.md | 10 +-
.../s3guard/AbstractS3GuardToolTestBase.java | 31 +
.../s3a/s3guard/ITestDynamoDBMetadataStore.java | 69 +-
.../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 3 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 21 +-
.../fs/azurebfs/services/AbfsInputStream.java | 9 +
.../fs/azurebfs/services/AbfsOutputStream.java | 16 +-
.../azurebfs/AbstractAbfsIntegrationTest.java | 4 +
.../hadoop/fs/azurebfs/ITestAbfsClient.java | 2 +-
.../azurebfs/ITestAzureBlobFileSystemE2E.java | 51 +-
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 96 +
.../constants/TestConfigurationKeys.java | 1 +
.../namenode/ITestProvidedImplementation.java | 139 +-
.../hadoop/yarn/api/records/Resource.java | 5 -
.../distributedshell/ApplicationMaster.java | 58 +
.../applications/distributedshell/Client.java | 77 +-
.../distributedshell/TestDistributedShell.java | 66 +-
.../src/test/resources/a.txt | 15 +
.../src/test/resources/b.txt | 15 +
.../component/instance/ComponentInstance.java | 76 +-
.../instance/TestComponentInstance.java | 36 +
.../yarn/submarine/client/cli/CliConstants.java | 1 +
.../yarn/submarine/client/cli/CliUtils.java | 23 +-
.../yarn/submarine/client/cli/RunJobCli.java | 18 +
.../client/cli/param/Localization.java | 133 +
.../client/cli/param/RunJobParameters.java | 15 +
.../client/cli/param/package-info.java | 19 +
.../common/conf/SubmarineConfiguration.java | 15 +
.../fs/DefaultRemoteDirectoryManager.java | 70 +-
.../common/fs/RemoteDirectoryManager.java | 18 +-
.../common/FSBasedSubmarineStorageImpl.java | 8 +-
.../yarnservice/YarnServiceJobSubmitter.java | 283 +-
.../src/site/markdown/QuickStart.md | 26 +
.../yarnservice/TestYarnServiceRunJobCli.java | 795 +-
.../submarine/common/MockClientContext.java | 8 +-
.../common/fs/MockRemoteDirectoryManager.java | 90 +-
.../yarn/client/api/impl/YarnClientImpl.java | 5 +-
.../hadoop/yarn/client/cli/ApplicationCLI.java | 5 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../resource/DefaultResourceCalculator.java | 2 +-
.../resource/DominantResourceCalculator.java | 4 +-
.../yarn/util/resource/ResourceCalculator.java | 18 +
...ustomResourceTypesConfigurationProvider.java | 186 +
.../util/resource/TestResourceCalculator.java | 26 +-
.../yarn/util/resource/TestResourceUtils.java | 17 -
.../timeline/KeyValueBasedTimelineStore.java | 98 +-
.../server/timeline/MemoryTimelineStore.java | 37 +-
.../timeline/TimelineStoreMapAdapter.java | 9 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 1 +
.../nodemanager/LinuxContainerExecutor.java | 7 +-
.../api/deviceplugin/DevicePluginScheduler.java | 38 +
.../container/ContainerImpl.java | 23 +-
.../linux/privileged/PrivilegedOperation.java | 2 +-
.../CGroupsBlkioResourceHandlerImpl.java | 5 +
.../CGroupsCpuResourceHandlerImpl.java | 5 +
.../linux/resources/CGroupsHandlerImpl.java | 14 +-
.../CGroupsMemoryResourceHandlerImpl.java | 5 +
.../NetworkPacketTaggingHandlerImpl.java | 5 +
.../linux/resources/ResourceHandlerChain.java | 6 +
.../linux/resources/ResourceHandlerModule.java | 37 +-
.../TrafficControlBandwidthHandlerImpl.java | 5 +
.../resources/fpga/FpgaResourceHandlerImpl.java | 14 +-
.../resources/gpu/GpuResourceAllocator.java | 5 +
.../resources/gpu/GpuResourceHandlerImpl.java | 15 +-
.../resources/numa/NumaResourceHandlerImpl.java | 12 +-
.../runtime/DefaultLinuxContainerRuntime.java | 103 +-
.../runtime/DockerLinuxContainerRuntime.java | 6 +-
.../linux/runtime/docker/DockerClient.java | 8 +-
.../monitor/ContainerMetrics.java | 8 +-
.../monitor/ContainerStopMonitoringEvent.java | 12 +
.../monitor/ContainersMonitorImpl.java | 6 +-
.../resourceplugin/ResourcePluginManager.java | 47 +-
.../deviceframework/AssignedDevice.java | 78 +
.../deviceframework/DeviceMappingManager.java | 73 +-
.../deviceframework/DevicePluginAdapter.java | 25 +-
.../DeviceResourceHandlerImpl.java | 21 +-
.../resourceplugin/fpga/FpgaResourcePlugin.java | 5 +
.../resourceplugin/gpu/GpuResourcePlugin.java | 5 +
.../executor/ContainerExecContext.java | 28 +-
.../webapp/ContainerShellWebSocket.java | 11 +
.../nodemanager/webapp/TerminalServlet.java | 49 +
.../server/nodemanager/webapp/WebServer.java | 9 +-
.../webapp/dao/NMDeviceResourceInfo.java | 56 +
.../impl/container-executor.c | 118 +-
.../impl/container-executor.h | 15 +-
.../main/native/container-executor/impl/main.c | 37 +-
.../test/test-container-executor.c | 50 +
.../src/main/resources/TERMINAL/css/style.css | 33 +
.../main/resources/TERMINAL/terminal.template | 124 +
.../TERMINAL/xterm/dist/addons/attach/attach.js | 104 +
.../xterm/dist/addons/attach/attach.js.map | 1 +
.../TERMINAL/xterm/dist/addons/fit/fit.js | 51 +
.../TERMINAL/xterm/dist/addons/fit/fit.js.map | 1 +
.../xterm/dist/addons/fullscreen/fullscreen.css | 10 +
.../xterm/dist/addons/fullscreen/fullscreen.js | 27 +
.../dist/addons/fullscreen/fullscreen.js.map | 1 +
.../TERMINAL/xterm/dist/addons/search/search.js | 166 +
.../xterm/dist/addons/search/search.js.map | 1 +
.../xterm/dist/addons/terminado/terminado.js | 69 +
.../dist/addons/terminado/terminado.js.map | 1 +
.../xterm/dist/addons/webLinks/webLinks.js | 41 +
.../xterm/dist/addons/webLinks/webLinks.js.map | 1 +
.../dist/addons/winptyCompat/winptyCompat.js | 31 +
.../addons/winptyCompat/winptyCompat.js.map | 1 +
.../TERMINAL/xterm/dist/addons/zmodem/zmodem.js | 45 +
.../xterm/dist/addons/zmodem/zmodem.js.map | 1 +
.../resources/TERMINAL/xterm/dist/xterm.css | 164 +
.../main/resources/TERMINAL/xterm/dist/xterm.js | 8907 ++++++++++++++++++
.../resources/TERMINAL/xterm/dist/xterm.js.map | 1 +
.../resources/fpga/TestFpgaResourceHandler.java | 5 +-
.../resources/gpu/TestGpuResourceHandler.java | 5 +-
.../linux/runtime/docker/TestDockerClient.java | 46 +
.../monitor/TestContainerMetrics.java | 6 +-
.../TestResourcePluginManager.java | 28 +
.../deviceframework/FakeTestDevicePlugin1.java | 18 +-
.../deviceframework/FakeTestDevicePlugin5.java | 56 +
.../TestDevicePluginAdapter.java | 137 +-
.../nodemanager/webapp/TestNMWebTerminal.java | 124 +
.../scheduler/capacity/CSQueueMetrics.java | 39 +
.../scheduler/capacity/CSQueueUtils.java | 17 +
.../scheduler/capacity/LeafQueue.java | 4 +
.../scheduler/capacity/ParentQueue.java | 4 +
.../scheduler/fair/FSParentQueue.java | 7 +-
.../scheduler/fair/FairScheduler.java | 52 +-
.../fair/FairSchedulerConfiguration.java | 95 +-
.../scheduler/TestSchedulerUtils.java | 51 +-
.../capacity/TestCapacityScheduler.java | 44 +
.../capacity/TestContainerAllocation.java | 5 +-
.../fair/TestAllocationFileLoaderService.java | 7 +-
.../scheduler/fair/TestFairScheduler.java | 102 +-
.../fair/TestFairSchedulerConfiguration.java | 320 +-
...TestRMWebServicesAppCustomResourceTypes.java | 2 +-
...estRMWebServicesAppsCustomResourceTypes.java | 2 +-
...RMWebServicesCustomResourceTypesCommons.java | 3 +-
...ustomResourceTypesConfigurationProvider.java | 140 -
...ervicesFairSchedulerCustomResourceTypes.java | 5 +-
.../timeline/LevelDBCacheTimelineStore.java | 14 +-
.../TestEntityGroupFSTimelineStore.java | 6 +-
.../src/site/markdown/SecureContainer.md | 3 +
.../markdown/yarn-service/YarnServiceAPI.md | 57 +-
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 41 +
.../main/webapp/app/components/timeline-view.js | 19 +-
.../src/main/webapp/app/initializers/env.js | 1 +
.../src/main/webapp/app/initializers/loader.js | 26 +
.../app/models/yarn-component-instance.js | 10 +
.../main/webapp/app/models/yarn-container.js | 2 +-
.../app/models/yarn-timeline-container.js | 2 +-
.../src/main/webapp/app/router.js | 2 +-
.../main/webapp/app/routes/yarn-app-attempt.js | 5 +
.../app/routes/yarn-component-instance/info.js | 5 +
.../templates/components/container-table.hbs | 6 +
.../app/templates/components/timeline-view.hbs | 2 +-
.../webapp/app/templates/yarn-app-attempt.hbs | 1 +
.../templates/yarn-component-instance/info.hbs | 4 +
340 files changed, 22020 insertions(+), 1980 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d891a54/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d891a54/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d891a54/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d891a54/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/22] hadoop git commit: YARN-9089. Add Terminal Link to Service
component instance page for UI2. Contributed by Eric Yang
Posted by sh...@apache.org.
YARN-9089. Add Terminal Link to Service component instance page for UI2. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aab31097
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aab31097
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aab31097
Branch: refs/heads/HDFS-12943
Commit: aab310978f64d57cf414de095e6195cbc4d0ed55
Parents: 1ea29b7
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Dec 14 18:38:03 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Fri Dec 14 18:38:03 2018 -0800
----------------------------------------------------------------------
.../src/main/webapp/app/initializers/env.js | 1 +
.../src/main/webapp/app/initializers/loader.js | 26 ++++++++++++++++++++
.../app/models/yarn-component-instance.js | 10 ++++++++
.../app/routes/yarn-component-instance/info.js | 5 ++++
.../templates/yarn-component-instance/info.hbs | 4 +++
5 files changed, 46 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab31097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/env.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/env.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/env.js
index c613593..bb9329e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/env.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/env.js
@@ -21,6 +21,7 @@ export function initialize( application ) {
application.inject('controller', 'env', 'service:env');
application.inject('route', 'env', 'service:env');
application.inject('adapter', 'env', 'service:env');
+ application.inject('model', 'env', 'service:env');
}
export default {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab31097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
index befff8f..e695ea6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
@@ -110,6 +110,29 @@ function getClusterIdFromYARN(rmhost, application) {
return clusterId;
}
+function getNodeManagerPort(rmhost, application) {
+ var httpUrl = window.location.protocol + "//" +
+ (ENV.hosts.localBaseAddress ? ENV.hosts.localBaseAddress + '/' : '') + rmhost
+ + ":" + window.location.port + "/conf?name=yarn.nodemanager.webapp.address";
+ var port = "8042";
+ $.ajax({
+ type: 'GET',
+ dataType: 'json',
+ async: false,
+ context: this,
+ url: httpUrl,
+ success: function(data) {
+ port = data.property.value.split(":")[1];
+ application.advanceReadiness();
+ },
+ error: function() {
+ port = "8042";
+ application.advanceReadiness();
+ }
+ });
+ return port;
+}
+
function updateConfigs(application) {
var hostname = window.location.hostname;
var rmhost = hostname + (window.location.port ? ':' + window.location.port: '') +
@@ -134,6 +157,9 @@ function updateConfigs(application) {
var clusterIdFromYARN = getClusterIdFromYARN(rmhost, application);
ENV.clusterId = clusterIdFromYARN;
+ var nodeManagerPort = getNodeManagerPort(rmhost, application);
+ ENV.nodeManagerPort = nodeManagerPort;
+
if(!ENV.hosts.timelineWebAddress) {
var timelinehost = "";
$.ajax({
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab31097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-component-instance.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-component-instance.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-component-instance.js
index 532fc55..e4b6a9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-component-instance.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-component-instance.js
@@ -47,5 +47,15 @@ export default DS.Model.extend({
return Converter.timeStampToDate(timestamp);
}
return 'N/A';
+ }),
+
+ termLink: Ember.computed('node', 'containerId', function() {
+ var protocol = window.location.protocol;
+ var node = this.get('node');
+ var port = this.get('env.app.nodeManagerPort');
+ var containerId = this.get('containerId');
+ var url = protocol + "//" + node + ":" + port +
+ "/terminal/terminal.template?container=" + containerId;
+ return url;
})
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab31097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
index a67324a..71c2893 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-component-instance/info.js
@@ -39,6 +39,11 @@ export default AbstractRoute.extend({
});
},
+ afterModel(model) {
+ const appContrl = this.controllerFor('application');
+ model.userInfo = appContrl.get('userInfo');
+ },
+
unloadAll() {
this.store.unloadAll('yarn-component-instance');
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab31097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
index 1b9d04a..c9157b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
@@ -61,6 +61,10 @@
<td>Exit Status Code</td>
<td>{{check-availability model.container.exitStatusCode}}</td>
</tr>
+ <tr>
+ <td>Terminal</td>
+ <td><a href="{{model.container.termLink}}&user.name={{model.userInfo.requestedUser}}" target="_blank">Link</a></td>
+ </tr>
</tbody>
</table>
</div>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/22] hadoop git commit: HADOOP-15969. ABFS: getNamespaceEnabled
can fail blocking user access thru ACLs.
Posted by sh...@apache.org.
HADOOP-15969. ABFS: getNamespaceEnabled can fail blocking user access thru ACLs.
Contributed by Da Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2523d81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2523d81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2523d81
Branch: refs/heads/HDFS-12943
Commit: b2523d8100844338e073531c47666d744a101caf
Parents: 788e747
Author: Da Zhou <da...@microsoft.com>
Authored: Mon Dec 17 11:04:40 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Dec 17 11:04:40 2018 +0000
----------------------------------------------------------------------
.../fs/azurebfs/AzureBlobFileSystemStore.java | 19 ++--
.../azurebfs/AbstractAbfsIntegrationTest.java | 4 +
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 96 ++++++++++++++++++++
.../constants/TestConfigurationKeys.java | 1 +
4 files changed, 114 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 5f1692f..5d70b77 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.azurebfs;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
+import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -181,12 +182,18 @@ public class AzureBlobFileSystemStore {
public boolean getIsNamespaceEnabled() throws AzureBlobFileSystemException {
if (!isNamespaceEnabledSet) {
- LOG.debug("getFilesystemProperties for filesystem: {}",
- client.getFileSystem());
-
- final AbfsRestOperation op = client.getFilesystemProperties();
- isNamespaceEnabled = Boolean.parseBoolean(
- op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_NAMESPACE_ENABLED));
+ LOG.debug("Get root ACL status");
+ try {
+ client.getAclStatus(AbfsHttpConstants.FORWARD_SLASH + AbfsHttpConstants.ROOT_PATH);
+ isNamespaceEnabled = true;
+ } catch (AbfsRestOperationException ex) {
+ // Get ACL status is a HEAD request, its response doesn't contain errorCode
+ // So can only rely on its status code to determine its account type.
+ if (HttpURLConnection.HTTP_BAD_REQUEST != ex.getStatusCode()) {
+ throw ex;
+ }
+ isNamespaceEnabled = false;
+ }
isNamespaceEnabledSet = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index 382cd7f..cb9549d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -262,6 +262,10 @@ public abstract class AbstractAbfsIntegrationTest extends
return this.authType;
}
+ public String getAbfsScheme() {
+ return this.abfsScheme;
+ }
+
protected boolean isIPAddress() {
return isIPAddress;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
new file mode 100644
index 0000000..b9a2315
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.junit.Assume;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.services.AuthType;
+
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test getIsNamespaceEnabled call.
+ */
+public class ITestGetNameSpaceEnabled extends AbstractAbfsIntegrationTest {
+
+ private boolean isUsingXNSAccount;
+ public ITestGetNameSpaceEnabled() throws Exception {
+ isUsingXNSAccount = getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
+ }
+
+ @Test
+ public void testXNSAccount() throws IOException {
+ Assume.assumeTrue("Skip this test because the account being used for test is a non XNS account",
+ isUsingXNSAccount);
+ assertTrue("Expecting getIsNamespaceEnabled() return true",
+ getFileSystem().getIsNamespaceEnabled());
+ }
+
+ @Test
+ public void testNonXNSAccount() throws IOException {
+ Assume.assumeFalse("Skip this test because the account being used for test is a XNS account",
+ isUsingXNSAccount);
+ assertFalse("Expecting getIsNamespaceEnabled() return false",
+ getFileSystem().getIsNamespaceEnabled());
+ }
+
+ @Test
+ public void testFailedRequestWhenFSNotExist() throws Exception {
+ AbfsConfiguration config = this.getConfiguration();
+ config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false);
+ String testUri = this.getTestUrl();
+ String nonExistingFsUrl = getAbfsScheme() + "://" + UUID.randomUUID()
+ + testUri.substring(testUri.indexOf("@"));
+ AzureBlobFileSystem fs = this.getFileSystem(nonExistingFsUrl);
+
+ intercept(AbfsRestOperationException.class,
+ "\"The specified filesystem does not exist.\", 404",
+ ()-> {
+ fs.getIsNamespaceEnabled();
+ });
+ }
+
+ @Test
+ public void testFailedRequestWhenCredentialsNotCorrect() throws Exception {
+ Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
+ Configuration config = this.getRawConfiguration();
+ config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false);
+ String accountName = this.getAccountName();
+ String configkKey = FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "." + accountName;
+ // Provide a wrong sharedKey
+ String secret = config.get(configkKey);
+ secret = (char) (secret.charAt(0) + 1) + secret.substring(1);
+ config.set(configkKey, secret);
+
+ AzureBlobFileSystem fs = this.getFileSystem(config);
+ intercept(AbfsRestOperationException.class,
+ "\"Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.\", 403",
+ ()-> {
+ fs.getIsNamespaceEnabled();
+ });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2523d81/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
index 5565a49..fbd13fe 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
@@ -26,6 +26,7 @@ public final class TestConfigurationKeys {
public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name";
public static final String FS_AZURE_ACCOUNT_KEY = "fs.azure.account.key";
public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs";
+ public static final String FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT = "fs.azure.test.namespace.enabled";
public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id";
public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_SECRET = "fs.azure.account.oauth2.contributor.client.secret";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/22] hadoop git commit: HADOOP-15972 ABFS: reduce list page size
to to 500.
Posted by sh...@apache.org.
HADOOP-15972 ABFS: reduce list page size to to 500.
Contributed by Da Zhou.
(cherry picked from commit d09dbcc8fb6889e51d4d4c914eb6159e40845e9f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62df6073
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62df6073
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62df6073
Branch: refs/heads/HDFS-12943
Commit: 62df60737c828afc3d1526973a24c6779a1ca9e6
Parents: b2523d8
Author: Da Zhou <da...@microsoft.com>
Authored: Mon Dec 17 11:10:22 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Dec 17 11:10:22 2018 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 2 +-
.../test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62df6073/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 5d70b77..b2bd64f 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -107,7 +107,7 @@ public class AzureBlobFileSystemStore {
private String primaryUserGroup;
private static final String DATE_TIME_PATTERN = "E, dd MMM yyyy HH:mm:ss 'GMT'";
private static final String XMS_PROPERTIES_ENCODING = "ISO-8859-1";
- private static final int LIST_MAX_RESULTS = 5000;
+ private static final int LIST_MAX_RESULTS = 500;
private static final int DELETE_DIRECTORY_TIMEOUT_MILISECONDS = 180000;
private static final int RENAME_TIMEOUT_MILISECONDS = 180000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62df6073/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
index f024f25..0f231d8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsClient.java
@@ -28,7 +28,7 @@ import org.junit.Test;
* Test continuation token which has equal sign.
*/
public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
- private static final int LIST_MAX_RESULTS = 5000;
+ private static final int LIST_MAX_RESULTS = 500;
public ITestAbfsClient() throws Exception {
super();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/22] hadoop git commit: HDFS-14088. RequestHedgingProxyProvider
can throw NullPointerException when failover due to no lock on
currentUsedProxy. Contributed by Yuxuan Wang.
Posted by sh...@apache.org.
HDFS-14088. RequestHedgingProxyProvider can throw NullPointerException when failover due to no lock on currentUsedProxy. Contributed by Yuxuan Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f858f185
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f858f185
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f858f185
Branch: refs/heads/HDFS-12943
Commit: f858f1855455348f5ac517ebd96c2bbc3bc97489
Parents: cb3382f
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Dec 14 11:04:36 2018 -0800
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Dec 14 11:04:36 2018 -0800
----------------------------------------------------------------------
.../ha/RequestHedgingProxyProvider.java | 209 ++++++++++---------
.../ha/TestRequestHedgingProxyProvider.java | 60 ++++++
2 files changed, 174 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f858f185/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
index c074bb8..9011b25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
@@ -58,6 +58,8 @@ public class RequestHedgingProxyProvider<T> extends
class RequestHedgingInvocationHandler implements InvocationHandler {
final Map<String, ProxyInfo<T>> targetProxies;
+ // Proxy of the active nn
+ private volatile ProxyInfo<T> currentUsedProxy = null;
public RequestHedgingInvocationHandler(
Map<String, ProxyInfo<T>> targetProxies) {
@@ -79,104 +81,118 @@ public class RequestHedgingProxyProvider<T> extends
public Object
invoke(Object proxy, final Method method, final Object[] args)
throws Throwable {
- if (currentUsedProxy != null) {
- try {
- Object retVal = method.invoke(currentUsedProxy.proxy, args);
- LOG.debug("Invocation successful on [{}]",
- currentUsedProxy.proxyInfo);
- return retVal;
- } catch (InvocationTargetException ex) {
- Exception unwrappedException = unwrapInvocationTargetException(ex);
- logProxyException(unwrappedException, currentUsedProxy.proxyInfo);
- LOG.trace("Unsuccessful invocation on [{}]",
- currentUsedProxy.proxyInfo);
- throw unwrappedException;
- }
- }
- Map<Future<Object>, ProxyInfo<T>> proxyMap = new HashMap<>();
- int numAttempts = 0;
+ // Need double check locking to guarantee thread-safe since
+ // currentUsedProxy is lazily initialized.
+ if (currentUsedProxy == null) {
+ synchronized (this) {
+ if (currentUsedProxy == null) {
+ Map<Future<Object>, ProxyInfo<T>> proxyMap = new HashMap<>();
+ int numAttempts = 0;
- ExecutorService executor = null;
- CompletionService<Object> completionService;
- try {
- // Optimization : if only 2 proxies are configured and one had failed
- // over, then we dont need to create a threadpool etc.
- targetProxies.remove(toIgnore);
- if (targetProxies.size() == 0) {
- LOG.trace("No valid proxies left");
- throw new RemoteException(IOException.class.getName(),
- "No valid proxies left. All NameNode proxies have failed over.");
- }
- if (targetProxies.size() == 1) {
- ProxyInfo<T> proxyInfo = targetProxies.values().iterator().next();
- try {
- currentUsedProxy = proxyInfo;
- Object retVal = method.invoke(proxyInfo.proxy, args);
- LOG.debug("Invocation successful on [{}]",
- currentUsedProxy.proxyInfo);
- return retVal;
- } catch (InvocationTargetException ex) {
- Exception unwrappedException = unwrapInvocationTargetException(ex);
- logProxyException(unwrappedException, currentUsedProxy.proxyInfo);
- LOG.trace("Unsuccessful invocation on [{}]",
- currentUsedProxy.proxyInfo);
- throw unwrappedException;
- }
- }
- executor = Executors.newFixedThreadPool(proxies.size());
- completionService = new ExecutorCompletionService<>(executor);
- for (final Map.Entry<String, ProxyInfo<T>> pEntry :
- targetProxies.entrySet()) {
- Callable<Object> c = new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- LOG.trace("Invoking method {} on proxy {}", method,
- pEntry.getValue().proxyInfo);
- return method.invoke(pEntry.getValue().proxy, args);
- }
- };
- proxyMap.put(completionService.submit(c), pEntry.getValue());
- numAttempts++;
- }
+ ExecutorService executor = null;
+ CompletionService<Object> completionService;
+ try {
+ // Optimization : if only 2 proxies are configured and one had
+ // failed
+ // over, then we dont need to create a threadpool etc.
+ targetProxies.remove(toIgnore);
+ if (targetProxies.size() == 0) {
+ LOG.trace("No valid proxies left");
+ throw new RemoteException(IOException.class.getName(),
+ "No valid proxies left. "
+ + "All NameNode proxies have failed over.");
+ }
+ if (targetProxies.size() == 1) {
+ ProxyInfo<T> proxyInfo =
+ targetProxies.values().iterator().next();
+ try {
+ currentUsedProxy = proxyInfo;
+ Object retVal = method.invoke(proxyInfo.proxy, args);
+ LOG.debug("Invocation successful on [{}]",
+ currentUsedProxy.proxyInfo);
+ return retVal;
+ } catch (InvocationTargetException ex) {
+ Exception unwrappedException =
+ unwrapInvocationTargetException(ex);
+ logProxyException(unwrappedException,
+ currentUsedProxy.proxyInfo);
+ LOG.trace("Unsuccessful invocation on [{}]",
+ currentUsedProxy.proxyInfo);
+ throw unwrappedException;
+ }
+ }
+ executor = Executors.newFixedThreadPool(proxies.size());
+ completionService = new ExecutorCompletionService<>(executor);
+ for (final Map.Entry<String, ProxyInfo<T>> pEntry : targetProxies
+ .entrySet()) {
+ Callable<Object> c = new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ LOG.trace("Invoking method {} on proxy {}", method,
+ pEntry.getValue().proxyInfo);
+ return method.invoke(pEntry.getValue().proxy, args);
+ }
+ };
+ proxyMap.put(completionService.submit(c), pEntry.getValue());
+ numAttempts++;
+ }
- Map<String, Exception> badResults = new HashMap<>();
- while (numAttempts > 0) {
- Future<Object> callResultFuture = completionService.take();
- Object retVal;
- try {
- currentUsedProxy = proxyMap.get(callResultFuture);
- retVal = callResultFuture.get();
- LOG.debug("Invocation successful on [{}]",
- currentUsedProxy.proxyInfo);
- return retVal;
- } catch (ExecutionException ex) {
- Exception unwrappedException = unwrapExecutionException(ex);
- ProxyInfo<T> tProxyInfo = proxyMap.get(callResultFuture);
- logProxyException(unwrappedException, tProxyInfo.proxyInfo);
- badResults.put(tProxyInfo.proxyInfo, unwrappedException);
- LOG.trace("Unsuccessful invocation on [{}]", tProxyInfo.proxyInfo);
- numAttempts--;
- }
- }
+ Map<String, Exception> badResults = new HashMap<>();
+ while (numAttempts > 0) {
+ Future<Object> callResultFuture = completionService.take();
+ Object retVal;
+ try {
+ currentUsedProxy = proxyMap.get(callResultFuture);
+ retVal = callResultFuture.get();
+ LOG.debug("Invocation successful on [{}]",
+ currentUsedProxy.proxyInfo);
+ return retVal;
+ } catch (ExecutionException ex) {
+ Exception unwrappedException = unwrapExecutionException(ex);
+ ProxyInfo<T> tProxyInfo = proxyMap.get(callResultFuture);
+ logProxyException(unwrappedException, tProxyInfo.proxyInfo);
+ badResults.put(tProxyInfo.proxyInfo, unwrappedException);
+ LOG.trace("Unsuccessful invocation on [{}]",
+ tProxyInfo.proxyInfo);
+ numAttempts--;
+ }
+ }
- // At this point we should have All bad results (Exceptions)
- // Or should have returned with successful result.
- if (badResults.size() == 1) {
- throw badResults.values().iterator().next();
- } else {
- throw new MultiException(badResults);
- }
- } finally {
- if (executor != null) {
- LOG.trace("Shutting down threadpool executor");
- executor.shutdownNow();
+ // At this point we should have All bad results (Exceptions)
+ // Or should have returned with successful result.
+ if (badResults.size() == 1) {
+ throw badResults.values().iterator().next();
+ } else {
+ throw new MultiException(badResults);
+ }
+ } finally {
+ if (executor != null) {
+ LOG.trace("Shutting down threadpool executor");
+ executor.shutdownNow();
+ }
+ }
+ }
}
}
+ // Because the above synchronized block will return or throw an exception,
+ // so we don't need to do any check to prevent the first initialized
+ // thread from stepping to following codes.
+ try {
+ Object retVal = method.invoke(currentUsedProxy.proxy, args);
+ LOG.debug("Invocation successful on [{}]", currentUsedProxy.proxyInfo);
+ return retVal;
+ } catch (InvocationTargetException ex) {
+ Exception unwrappedException = unwrapInvocationTargetException(ex);
+ logProxyException(unwrappedException, currentUsedProxy.proxyInfo);
+ LOG.trace("Unsuccessful invocation on [{}]",
+ currentUsedProxy.proxyInfo);
+ throw unwrappedException;
+ }
}
}
-
- private volatile ProxyInfo<T> currentUsedProxy = null;
+ /** A proxy wrapping {@link RequestHedgingInvocationHandler}. */
+ private ProxyInfo<T> currentUsedHandler = null;
private volatile String toIgnore = null;
public RequestHedgingProxyProvider(Configuration conf, URI uri,
@@ -187,8 +203,8 @@ public class RequestHedgingProxyProvider<T> extends
@SuppressWarnings("unchecked")
@Override
public synchronized ProxyInfo<T> getProxy() {
- if (currentUsedProxy != null) {
- return currentUsedProxy;
+ if (currentUsedHandler != null) {
+ return currentUsedHandler;
}
Map<String, ProxyInfo<T>> targetProxyInfos = new HashMap<>();
StringBuilder combinedInfo = new StringBuilder("[");
@@ -203,13 +219,16 @@ public class RequestHedgingProxyProvider<T> extends
RequestHedgingInvocationHandler.class.getClassLoader(),
new Class<?>[]{xface},
new RequestHedgingInvocationHandler(targetProxyInfos));
- return new ProxyInfo<T>(wrappedProxy, combinedInfo.toString());
+ currentUsedHandler =
+ new ProxyInfo<T>(wrappedProxy, combinedInfo.toString());
+ return currentUsedHandler;
}
@Override
public synchronized void performFailover(T currentProxy) {
- toIgnore = this.currentUsedProxy.proxyInfo;
- this.currentUsedProxy = null;
+ toIgnore = ((RequestHedgingInvocationHandler) Proxy.getInvocationHandler(
+ currentUsedHandler.proxy)).currentUsedProxy.proxyInfo;
+ this.currentUsedHandler = null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f858f185/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
index 5c33ef6..fc81a09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.junit.Assert;
@@ -634,6 +635,65 @@ public class TestRequestHedgingProxyProvider {
Mockito.verify(standby).getStats();
}
+ /**
+ * HDFS-14088, we first make a successful RPC call, so
+ * RequestHedgingInvocationHandler#currentUsedProxy will be assigned to the
+ * delayMock. Then: <br/>
+ * 1. We start a thread which sleep for 1 sec and call
+ * RequestHedgingProxyProvider#performFailover() <br/>
+ * 2. We make an RPC call again, the call will sleep for 2 sec and throw an
+ * exception for test.<br/>
+ * 3. RequestHedgingInvocationHandler#invoke() will catch the exception and
+ * log RequestHedgingInvocationHandler#currentUsedProxy. Before patch, there
+ * will throw NullPointException.
+ * @throws Exception
+ */
+ @Test
+ public void testHedgingMultiThreads() throws Exception {
+ final AtomicInteger counter = new AtomicInteger(0);
+ final ClientProtocol delayMock = Mockito.mock(ClientProtocol.class);
+ Mockito.when(delayMock.getStats()).thenAnswer(new Answer<long[]>() {
+ @Override
+ public long[] answer(InvocationOnMock invocation) throws Throwable {
+ int flag = counter.incrementAndGet();
+ Thread.sleep(2000);
+ if (flag == 1) {
+ return new long[]{1};
+ } else {
+ throw new IOException("Exception for test.");
+ }
+ }
+ });
+ final ClientProtocol badMock = Mockito.mock(ClientProtocol.class);
+ Mockito.when(badMock.getStats()).thenThrow(new IOException("Bad mock !!"));
+ final RequestHedgingProxyProvider<ClientProtocol> provider =
+ new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class,
+ createFactory(delayMock, badMock));
+ final ClientProtocol delayProxy = provider.getProxy().proxy;
+ long[] stats = delayProxy.getStats();
+ Assert.assertTrue(stats.length == 1);
+ Assert.assertEquals(1, stats[0]);
+ Assert.assertEquals(1, counter.get());
+
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ // Fail over between calling delayProxy.getStats() and throw
+ // exception.
+ Thread.sleep(1000);
+ provider.performFailover(delayProxy);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ };
+ t.start();
+ LambdaTestUtils.intercept(IOException.class, "Exception for test.",
+ delayProxy::getStats);
+ t.join();
+ }
+
private HAProxyFactory<ClientProtocol> createFactory(
ClientProtocol... protos) {
final Iterator<ClientProtocol> iterator =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/22] hadoop git commit: YARN-9091. Improve terminal message when
connection is refused. Contributed by Eric Yang
Posted by sh...@apache.org.
YARN-9091. Improve terminal message when connection is refused. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f622337
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f622337
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f622337
Branch: refs/heads/HDFS-12943
Commit: 4f622337eed5d7d956a952d2fa25e78615da1787
Parents: 42297f7
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Dec 14 19:55:50 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Fri Dec 14 19:55:50 2018 -0800
----------------------------------------------------------------------
.../src/main/resources/TERMINAL/terminal.template | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f622337/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL/terminal.template
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL/terminal.template b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL/terminal.template
index 004e8e27..4bff794 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL/terminal.template
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL/terminal.template
@@ -87,11 +87,25 @@
if (user != null) {
url = url + "?user.name=" + encodeURI(user);
}
+ document.title = getQueryVariable("container");
console.log(url);
var sock = new WebSocket(url);
sock.addEventListener('open', function () {
term.attach(sock, true, true);
});
+ sock.addEventListener('close', function(event) {
+ switch(event.code) {
+ case 1000:
+ term.write('Remote Connection closed.');
+ break;
+ case 1008:
+ term.write('Remote Connection refused.');
+ break;
+ case 1001:
+ term.write('Remote Connection going away.');
+ break;
+ }
+ });
term.open(container);
term.winptyCompatInit();
term.fit();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/22] hadoop git commit: HDFS-14096. [SPS] : Add Support for
Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena.
Posted by sh...@apache.org.
HDFS-14096. [SPS] : Add Support for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/788e7473
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/788e7473
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/788e7473
Branch: refs/heads/HDFS-12943
Commit: 788e7473a404fa074b3af522416ee3d2fae865a0
Parents: 04c0347
Author: Surendra Singh Lilhore <su...@apache.org>
Authored: Mon Dec 17 11:24:57 2018 +0530
Committer: Surendra Singh Lilhore <su...@apache.org>
Committed: Mon Dec 17 11:24:57 2018 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/fs/AbstractFileSystem.java | 10 ++++++++++
.../java/org/apache/hadoop/fs/FileContext.java | 18 ++++++++++++++++++
.../java/org/apache/hadoop/fs/FileSystem.java | 10 ++++++++++
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +++++
.../main/java/org/apache/hadoop/fs/FilterFs.java | 5 +++++
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 +++++
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 5 +++++
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 13 +++++++++++++
.../java/org/apache/hadoop/fs/viewfs/ViewFs.java | 12 ++++++++++++
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 ++
.../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 5 +++++
.../src/main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++
.../apache/hadoop/hdfs/DistributedFileSystem.java | 6 +-----
13 files changed, 96 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index 721f009..d1e50d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -1255,6 +1255,16 @@ public abstract class AbstractFileSystem {
}
/**
+ * Set the source path to satisfy storage policy.
+ * @param path The source path referring to either a directory or a file.
+ * @throws IOException
+ */
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ throw new UnsupportedOperationException(
+ getClass().getSimpleName() + " doesn't support satisfyStoragePolicy");
+ }
+
+ /**
* Set the storage policy for a given file or directory.
*
* @param path file or directory path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 084f8f1..e5438f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -2781,6 +2781,24 @@ public class FileContext {
}
/**
+ * Set the source path to satisfy storage policy.
+ * @param path The source path referring to either a directory or a file.
+ * @throws IOException
+ */
+ public void satisfyStoragePolicy(final Path path)
+ throws IOException {
+ final Path absF = fixRelativePart(path);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException {
+ fs.satisfyStoragePolicy(path);
+ return null;
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
* Set the storage policy for a given file or directory.
*
* @param path file or directory path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index fe4159b..5454cd0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -3086,6 +3086,16 @@ public abstract class FileSystem extends Configured
}
/**
+ * Set the source path to satisfy storage policy.
+ * @param path The source path referring to either a directory or a file.
+ * @throws IOException
+ */
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ throw new UnsupportedOperationException(
+ getClass().getSimpleName() + " doesn't support setStoragePolicy");
+ }
+
+ /**
* Set the storage policy for a given file or directory.
*
* @param src file or directory path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index f9bbfb1..a3e10b4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -646,6 +646,11 @@ public class FilterFileSystem extends FileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ fs.satisfyStoragePolicy(src);
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {
fs.setStoragePolicy(src, policyName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index b2a9aab..d6dc59b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -406,6 +406,11 @@ public abstract class FilterFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ myFs.satisfyStoragePolicy(path);
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {
myFs.setStoragePolicy(path, policyName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 8b90f53..b195be3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -450,6 +450,11 @@ class ChRootedFileSystem extends FilterFileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ super.satisfyStoragePolicy(fullPath(src));
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
super.setStoragePolicy(fullPath(src), policyName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index 6168df0..5e14be1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -399,6 +399,11 @@ class ChRootedFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ myFs.satisfyStoragePolicy(path);
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {
myFs.setStoragePolicy(fullPath(path), policyName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 9523070..bc73f1b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -811,6 +811,13 @@ public class ViewFileSystem extends FileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(src), true);
+ res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
true);
@@ -1246,6 +1253,12 @@ public class ViewFileSystem extends FileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path src) throws IOException {
+ checkPathIsSlash(src);
+ throw readOnlyMountTable("satisfyStoragePolicy", src);
+ }
+
+ @Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {
checkPathIsSlash(src);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 8366eac..1770476 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -752,6 +752,13 @@ public class ViewFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
+ }
+
+ @Override
public void setStoragePolicy(final Path path, final String policyName)
throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res =
@@ -1155,6 +1162,11 @@ public class ViewFs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(final Path path) throws IOException {
+ throw readOnlyMountTable("satisfyStoragePolicy", path);
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {
throw readOnlyMountTable("setStoragePolicy", path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 025b831..b3021a9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -213,6 +213,8 @@ public class TestHarFileSystem {
public void access(Path path, FsAction mode) throws IOException;
+ void satisfyStoragePolicy(Path src) throws IOException;
+
public void setStoragePolicy(Path src, String policyName)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index b6158dfc..b9453df 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -968,6 +968,11 @@ abstract public class ViewFileSystemBaseTest {
fsView.unsetStoragePolicy(new Path("/internalDir"));
}
+ @Test(expected = AccessControlException.class)
+ public void testInternalSatisfyStoragePolicy() throws IOException {
+ fsView.satisfyStoragePolicy(new Path("/internalDir"));
+ }
+
@Test(expected = NotInMountpointException.class)
public void testInternalgetStoragePolicy() throws IOException {
fsView.getStoragePolicy(new Path("/internalDir"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
index d07d5a2..290f2c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -490,6 +490,11 @@ public class Hdfs extends AbstractFileSystem {
}
@Override
+ public void satisfyStoragePolicy(Path path) throws IOException {
+ dfs.satisfyStoragePolicy(getUriPath(path));
+ }
+
+ @Override
public void setStoragePolicy(Path path, String policyName) throws IOException {
dfs.setStoragePolicy(getUriPath(path), policyName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/788e7473/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 7dd02bd..2b0b2c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2870,11 +2870,7 @@ public class DistributedFileSystem extends FileSystem
}
/**
- * Set the source path to satisfy storage policy. This API is non-recursive
- * in nature, i.e., if the source path is a directory then all the files
- * immediately under the directory would be considered for satisfying the
- * policy and the sub-directories if any under this path will be skipped.
- *
+ * Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org