You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by kk...@apache.org on 2017/10/03 21:23:18 UTC
[01/50] [abbrv] hadoop git commit: YARN-7240. Add more states and
transitions to stabilize the NM Container state machine. (Kartheek Muthyala
via asuresh) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-6592 a0e16d41d -> 229680a06 (forced update)
YARN-7240. Add more states and transitions to stabilize the NM Container state machine. (Kartheek Muthyala via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df800f6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df800f6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df800f6c
Branch: refs/heads/YARN-6592
Commit: df800f6cf3ea663daf4081ebe784808b08d9366d
Parents: 47011d7
Author: Arun Suresh <as...@apache.org>
Authored: Mon Sep 25 14:11:55 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Mon Sep 25 16:02:55 2017 -0700
----------------------------------------------------------------------
.../containermanager/ContainerManagerImpl.java | 41 +---
.../container/ContainerEventType.java | 6 +-
.../container/ContainerImpl.java | 174 ++++++++++++--
.../container/ContainerState.java | 3 +-
.../container/UpdateContainerTokenEvent.java | 86 +++++++
.../scheduler/ContainerScheduler.java | 114 ++++-----
.../UpdateContainerSchedulerEvent.java | 46 ++--
.../BaseContainerManagerTest.java | 2 +
.../containermanager/TestContainerManager.java | 229 ++++++++++++++++++-
.../TestContainerSchedulerQueuing.java | 101 ++++++++
10 files changed, 660 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index e497f62..d12892e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -144,7 +145,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredApplicationsState;
@@ -1251,29 +1251,6 @@ public class ContainerManagerImpl extends CompositeService implements
+ " [" + containerTokenIdentifier.getVersion() + "]");
}
- // Check container state
- org.apache.hadoop.yarn.server.nodemanager.
- containermanager.container.ContainerState currentState =
- container.getContainerState();
- EnumSet<org.apache.hadoop.yarn.server.nodemanager.containermanager
- .container.ContainerState> allowedStates = EnumSet.of(
- org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .ContainerState.RUNNING,
- org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .ContainerState.SCHEDULED,
- org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .ContainerState.LOCALIZING,
- org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .ContainerState.REINITIALIZING,
- org.apache.hadoop.yarn.server.nodemanager.containermanager.container
- .ContainerState.RELAUNCHING);
- if (!allowedStates.contains(currentState)) {
- throw RPCUtil.getRemoteException("Container " + containerId.toString()
- + " is in " + currentState.name() + " state."
- + " Resource can only be changed when a container is in"
- + " RUNNING or SCHEDULED state");
- }
-
// Check validity of the target resource.
Resource currentResource = container.getResource();
ExecutionType currentExecType =
@@ -1313,11 +1290,11 @@ public class ContainerManagerImpl extends CompositeService implements
this.readLock.lock();
try {
if (!serviceStopped) {
- // Dispatch message to ContainerScheduler to actually
+ // Dispatch message to Container to actually
// make the change.
- dispatcher.getEventHandler().handle(new UpdateContainerSchedulerEvent(
- container, containerTokenIdentifier, isResourceChange,
- isExecTypeUpdate, isIncrease));
+ dispatcher.getEventHandler().handle(new UpdateContainerTokenEvent(
+ container.getContainerId(), containerTokenIdentifier,
+ isResourceChange, isExecTypeUpdate, isIncrease));
} else {
throw new YarnException(
"Unable to change container resource as the NodeManager is "
@@ -1816,10 +1793,14 @@ public class ContainerManagerImpl extends CompositeService implements
if (container == null) {
throw new YarnException("Specified " + containerId + " does not exist!");
}
- if (!container.isRunning() || container.isReInitializing()) {
+ if (!container.isRunning() || container.isReInitializing()
+ || container.getContainerTokenIdentifier().getExecutionType()
+ == ExecutionType.OPPORTUNISTIC) {
throw new YarnException("Cannot perform " + op + " on [" + containerId
+ "]. Current state is [" + container.getContainerState() + ", " +
- "isReInitializing=" + container.isReInitializing() + "].");
+ "isReInitializing=" + container.isReInitializing() + "]. Container"
+ + " Execution Type is [" + container.getContainerTokenIdentifier()
+ .getExecutionType() + "].");
}
return container;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
index 1475435..e28b37d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
@@ -29,6 +29,7 @@ public enum ContainerEventType {
ROLLBACK_REINIT,
PAUSE_CONTAINER,
RESUME_CONTAINER,
+ UPDATE_CONTAINER_TOKEN,
// DownloadManager
CONTAINER_INITED,
@@ -42,5 +43,8 @@ public enum ContainerEventType {
CONTAINER_EXITED_WITH_FAILURE,
CONTAINER_KILLED_ON_REQUEST,
CONTAINER_PAUSED,
- CONTAINER_RESUMED
+ CONTAINER_RESUMED,
+
+ // Producer: ContainerScheduler
+ CONTAINER_TOKEN_UPDATED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 836e70e..705087b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -33,6 +33,8 @@ import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -308,8 +310,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.NEW, ContainerState.DONE,
ContainerEventType.KILL_CONTAINER, new KillOnNewTransition())
- .addTransition(ContainerState.NEW, ContainerState.DONE,
- ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
+ .addTransition(ContainerState.NEW, ContainerState.NEW,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN, new UpdateTransition())
// From LOCALIZING State
.addTransition(ContainerState.LOCALIZING,
@@ -325,8 +327,9 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.LOCALIZING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER,
new KillBeforeRunningTransition())
- .addTransition(ContainerState.LOCALIZING, ContainerState.KILLING,
- ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
+ .addTransition(ContainerState.LOCALIZING, ContainerState.LOCALIZING,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN, new UpdateTransition())
+
// From LOCALIZATION_FAILED State
.addTransition(ContainerState.LOCALIZATION_FAILED,
@@ -351,6 +354,9 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.LOCALIZATION_FAILED,
ContainerState.LOCALIZATION_FAILED,
ContainerEventType.RESOURCE_FAILED)
+ .addTransition(ContainerState.LOCALIZATION_FAILED,
+ ContainerState.LOCALIZATION_FAILED,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN, new UpdateTransition())
// From SCHEDULED State
.addTransition(ContainerState.SCHEDULED, ContainerState.RUNNING,
@@ -364,6 +370,9 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.SCHEDULED, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER,
new KillBeforeRunningTransition())
+ .addTransition(ContainerState.SCHEDULED, ContainerState.SCHEDULED,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
// From RUNNING State
.addTransition(ContainerState.RUNNING,
@@ -376,10 +385,16 @@ public class ContainerImpl implements Container {
ContainerState.EXITED_WITH_FAILURE),
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
new RetryFailureTransition())
- .addTransition(ContainerState.RUNNING, ContainerState.REINITIALIZING,
+ .addTransition(ContainerState.RUNNING,
+ EnumSet.of(ContainerState.RUNNING,
+ ContainerState.REINITIALIZING,
+ ContainerState.REINITIALIZING_AWAITING_KILL),
ContainerEventType.REINITIALIZE_CONTAINER,
new ReInitializeContainerTransition())
- .addTransition(ContainerState.RUNNING, ContainerState.REINITIALIZING,
+ .addTransition(ContainerState.RUNNING,
+ EnumSet.of(ContainerState.RUNNING,
+ ContainerState.REINITIALIZING,
+ ContainerState.REINITIALIZING_AWAITING_KILL),
ContainerEventType.ROLLBACK_REINIT,
new RollbackContainerTransition())
.addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
@@ -398,9 +413,16 @@ public class ContainerImpl implements Container {
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
new KilledExternallyTransition())
.addTransition(ContainerState.RUNNING, ContainerState.PAUSING,
- ContainerEventType.PAUSE_CONTAINER, new PauseContainerTransition())
+ ContainerEventType.PAUSE_CONTAINER, new PauseContainerTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
+
// From PAUSING State
+ .addTransition(ContainerState.PAUSING, ContainerState.PAUSING,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ new ResourceLocalizedWhileRunningTransition())
.addTransition(ContainerState.PAUSING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER, new KillTransition())
.addTransition(ContainerState.PAUSING, ContainerState.PAUSING,
@@ -420,6 +442,12 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.PAUSING, ContainerState.EXITED_WITH_FAILURE,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
new KilledExternallyTransition())
+ .addTransition(ContainerState.PAUSING, ContainerState.PAUSING,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ new ResourceLocalizedWhileRunningTransition())
+ .addTransition(ContainerState.PAUSING, ContainerState.PAUSING,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
// From PAUSED State
.addTransition(ContainerState.PAUSED, ContainerState.KILLING,
@@ -429,6 +457,10 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.PAUSED, ContainerState.PAUSED,
ContainerEventType.PAUSE_CONTAINER)
+ // This can happen during re-initialization.
+ .addTransition(ContainerState.PAUSED, ContainerState.PAUSED,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ new ResourceLocalizedWhileRunningTransition())
.addTransition(ContainerState.PAUSED, ContainerState.RESUMING,
ContainerEventType.RESUME_CONTAINER, new ResumeContainerTransition())
// In case something goes wrong then container will exit from the
@@ -444,6 +476,9 @@ public class ContainerImpl implements Container {
ContainerState.EXITED_WITH_SUCCESS,
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
new ExitedWithSuccessTransition(true))
+ .addTransition(ContainerState.PAUSED, ContainerState.PAUSED,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
// From RESUMING State
.addTransition(ContainerState.RESUMING, ContainerState.KILLING,
@@ -453,6 +488,10 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.RESUMING, ContainerState.RESUMING,
ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
UPDATE_DIAGNOSTICS_TRANSITION)
+ // This can happen during re-initialization
+ .addTransition(ContainerState.RESUMING, ContainerState.RESUMING,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ new ResourceLocalizedWhileRunningTransition())
// In case something goes wrong then container will exit from the
// RESUMING state
.addTransition(ContainerState.RESUMING,
@@ -467,6 +506,10 @@ public class ContainerImpl implements Container {
ContainerState.EXITED_WITH_SUCCESS,
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
new ExitedWithSuccessTransition(true))
+ .addTransition(ContainerState.RESUMING, ContainerState.RESUMING,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
+ // NOTE - We cannot get a PAUSE_CONTAINER while in RESUMING state.
// From REINITIALIZING State
.addTransition(ContainerState.REINITIALIZING,
@@ -478,7 +521,8 @@ public class ContainerImpl implements Container {
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
new ExitedWithFailureTransition(true))
.addTransition(ContainerState.REINITIALIZING,
- ContainerState.REINITIALIZING,
+ EnumSet.of(ContainerState.REINITIALIZING,
+ ContainerState.REINITIALIZING_AWAITING_KILL),
ContainerEventType.RESOURCE_LOCALIZED,
new ResourceLocalizedWhileReInitTransition())
.addTransition(ContainerState.REINITIALIZING, ContainerState.RUNNING,
@@ -490,12 +534,39 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.REINITIALIZING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER, new KillTransition())
- .addTransition(ContainerState.REINITIALIZING, ContainerState.KILLING,
- ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
+ .addTransition(ContainerState.REINITIALIZING, ContainerState.PAUSING,
+ ContainerEventType.PAUSE_CONTAINER, new PauseContainerTransition())
.addTransition(ContainerState.REINITIALIZING,
+ ContainerState.REINITIALIZING,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
+
+ // from REINITIALIZING_AWAITING_KILL
+ .addTransition(ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
+ new ExitedWithSuccessTransition(true))
+ .addTransition(ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ new ExitedWithFailureTransition(true))
+ .addTransition(ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ UPDATE_DIAGNOSTICS_TRANSITION)
+ .addTransition(ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerState.KILLING,
+ ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerState.SCHEDULED, ContainerEventType.PAUSE_CONTAINER)
+ .addTransition(ContainerState.REINITIALIZING_AWAITING_KILL,
ContainerState.SCHEDULED,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
new KilledForReInitializationTransition())
+ .addTransition(ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
// From RELAUNCHING State
.addTransition(ContainerState.RELAUNCHING, ContainerState.RUNNING,
@@ -511,6 +582,10 @@ public class ContainerImpl implements Container {
ContainerEventType.KILL_CONTAINER, new KillTransition())
.addTransition(ContainerState.RELAUNCHING, ContainerState.KILLING,
ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
+ .addTransition(ContainerState.RELAUNCHING, ContainerState.RELAUNCHING,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ new NotifyContainerSchedulerOfUpdateTransition())
+
// From CONTAINER_EXITED_WITH_SUCCESS State
.addTransition(ContainerState.EXITED_WITH_SUCCESS, ContainerState.DONE,
@@ -524,6 +599,10 @@ public class ContainerImpl implements Container {
ContainerState.EXITED_WITH_SUCCESS,
EnumSet.of(ContainerEventType.KILL_CONTAINER,
ContainerEventType.PAUSE_CONTAINER))
+ // No transition - assuming container is on its way to completion
+ .addTransition(ContainerState.EXITED_WITH_SUCCESS,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN)
// From EXITED_WITH_FAILURE State
.addTransition(ContainerState.EXITED_WITH_FAILURE, ContainerState.DONE,
@@ -537,6 +616,10 @@ public class ContainerImpl implements Container {
ContainerState.EXITED_WITH_FAILURE,
EnumSet.of(ContainerEventType.KILL_CONTAINER,
ContainerEventType.PAUSE_CONTAINER))
+ // No transition - assuming container is on its way to completion
+ .addTransition(ContainerState.EXITED_WITH_FAILURE,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN)
// From KILLING State.
.addTransition(ContainerState.KILLING,
@@ -572,6 +655,9 @@ public class ContainerImpl implements Container {
ContainerState.KILLING,
EnumSet.of(ContainerEventType.CONTAINER_LAUNCHED,
ContainerEventType.PAUSE_CONTAINER))
+ // No transition - assuming container is on its way to completion
+ .addTransition(ContainerState.KILLING, ContainerState.KILLING,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN)
// From CONTAINER_CLEANEDUP_AFTER_KILL State.
.addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
@@ -589,6 +675,10 @@ public class ContainerImpl implements Container {
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
ContainerEventType.PAUSE_CONTAINER))
+ // No transition - assuming container is on its way to completion
+ .addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
+ ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN)
// From DONE
.addTransition(ContainerState.DONE, ContainerState.DONE,
@@ -606,6 +696,9 @@ public class ContainerImpl implements Container {
EnumSet.of(ContainerEventType.RESOURCE_FAILED,
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE))
+ // No transition - assuming container is on its way to completion
+ .addTransition(ContainerState.DONE, ContainerState.DONE,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN)
// create the topology tables
.installTopology();
@@ -626,6 +719,7 @@ public class ContainerImpl implements Container {
case RUNNING:
case RELAUNCHING:
case REINITIALIZING:
+ case REINITIALIZING_AWAITING_KILL:
case EXITED_WITH_SUCCESS:
case EXITED_WITH_FAILURE:
case KILLING:
@@ -929,6 +1023,45 @@ public class ContainerImpl implements Container {
}
+ static class UpdateTransition extends ContainerTransition {
+ @Override
+ public void transition(
+ ContainerImpl container, ContainerEvent event) {
+ UpdateContainerTokenEvent updateEvent = (UpdateContainerTokenEvent)event;
+ // Update the container token
+ container.setContainerTokenIdentifier(updateEvent.getUpdatedToken());
+ if (updateEvent.isResourceChange()) {
+ try {
+ // Persist change in the state store.
+ container.context.getNMStateStore().storeContainerResourceChanged(
+ container.containerId,
+ container.getContainerTokenIdentifier().getVersion(),
+ container.getResource());
+ } catch (IOException e) {
+ LOG.warn("Could not store container [" + container.containerId
+ + "] resource change..", e);
+ }
+ }
+ }
+ }
+
+ static class NotifyContainerSchedulerOfUpdateTransition extends
+ UpdateTransition {
+ @Override
+ public void transition(
+ ContainerImpl container, ContainerEvent event) {
+
+ UpdateContainerTokenEvent updateEvent = (UpdateContainerTokenEvent)event;
+ // Save original token
+ ContainerTokenIdentifier originalToken =
+ container.containerTokenIdentifier;
+ super.transition(container, updateEvent);
+ container.dispatcher.getEventHandler().handle(
+ new UpdateContainerSchedulerEvent(container,
+ originalToken, updateEvent));
+ }
+ }
+
/**
* State transition when a NEW container receives the INIT_CONTAINER
* message.
@@ -1074,12 +1207,15 @@ public class ContainerImpl implements Container {
/**
* Transition to start the Re-Initialization process.
*/
- static class ReInitializeContainerTransition extends ContainerTransition {
+ static class ReInitializeContainerTransition implements
+ MultipleArcTransition<ContainerImpl, ContainerEvent, ContainerState> {
@SuppressWarnings("unchecked")
@Override
- public void transition(ContainerImpl container, ContainerEvent event) {
+ public ContainerState transition(
+ ContainerImpl container, ContainerEvent event) {
container.reInitContext = createReInitContext(container, event);
+ boolean resourcesPresent = false;
try {
// 'reInitContext.newResourceSet' can be
// a) current container resourceSet (In case of Restart)
@@ -1101,6 +1237,7 @@ public class ContainerImpl implements Container {
container.dispatcher.getEventHandler().handle(
new ContainersLauncherEvent(container,
ContainersLauncherEventType.CLEANUP_CONTAINER_FOR_REINIT));
+ resourcesPresent = true;
}
container.metrics.reInitingContainer();
NMAuditLogger.logSuccess(container.user,
@@ -1112,7 +1249,11 @@ public class ContainerImpl implements Container {
" re-initialization failure..", e);
container.addDiagnostics("Error re-initializing due to" +
"[" + e.getMessage() + "]");
+ return ContainerState.RUNNING;
}
+ return resourcesPresent ?
+ ContainerState.REINITIALIZING_AWAITING_KILL :
+ ContainerState.REINITIALIZING;
}
protected ReInitializationContext createReInitContext(
@@ -1164,11 +1305,14 @@ public class ContainerImpl implements Container {
* If all dependencies are met, then restart Container with new bits.
*/
static class ResourceLocalizedWhileReInitTransition
- extends ContainerTransition {
+ implements MultipleArcTransition
+ <ContainerImpl, ContainerEvent, ContainerState> {
+
@SuppressWarnings("unchecked")
@Override
- public void transition(ContainerImpl container, ContainerEvent event) {
+ public ContainerState transition(
+ ContainerImpl container, ContainerEvent event) {
ContainerResourceLocalizedEvent rsrcEvent =
(ContainerResourceLocalizedEvent) event;
container.reInitContext.newResourceSet.resourceLocalized(
@@ -1180,7 +1324,9 @@ public class ContainerImpl implements Container {
container.dispatcher.getEventHandler().handle(
new ContainersLauncherEvent(container,
ContainersLauncherEventType.CLEANUP_CONTAINER_FOR_REINIT));
+ return ContainerState.REINITIALIZING_AWAITING_KILL;
}
+ return ContainerState.REINITIALIZING;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
index 7c3fea8..5644d03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
@@ -20,7 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
public enum ContainerState {
NEW, LOCALIZING, LOCALIZATION_FAILED, SCHEDULED, RUNNING, RELAUNCHING,
- REINITIALIZING, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING,
+ REINITIALIZING, REINITIALIZING_AWAITING_KILL,
+ EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING,
CONTAINER_CLEANEDUP_AFTER_KILL, CONTAINER_RESOURCES_CLEANINGUP, DONE,
PAUSING, PAUSED, RESUMING
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/UpdateContainerTokenEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/UpdateContainerTokenEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/UpdateContainerTokenEvent.java
new file mode 100644
index 0000000..c9dc97e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/UpdateContainerTokenEvent.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+
+/**
+ * Update Event consumed by the Container.
+ */
+public class UpdateContainerTokenEvent extends ContainerEvent {
+ private final ContainerTokenIdentifier updatedToken;
+ private final boolean isResourceChange;
+ private final boolean isExecTypeUpdate;
+ private final boolean isIncrease;
+
+ /**
+ * Create Update event.
+ *
+ * @param cID Container Id.
+ * @param updatedToken Updated Container Token.
+ * @param isResourceChange Is Resource change.
+ * @param isExecTypeUpdate Is ExecutionType Update.
+ * @param isIncrease Is container increase.
+ */
+ public UpdateContainerTokenEvent(ContainerId cID,
+ ContainerTokenIdentifier updatedToken, boolean isResourceChange,
+ boolean isExecTypeUpdate, boolean isIncrease) {
+ super(cID, ContainerEventType.UPDATE_CONTAINER_TOKEN);
+ this.updatedToken = updatedToken;
+ this.isResourceChange = isResourceChange;
+ this.isExecTypeUpdate = isExecTypeUpdate;
+ this.isIncrease = isIncrease;
+ }
+
+ /**
+ * Update Container Token.
+ *
+ * @return Container Token.
+ */
+ public ContainerTokenIdentifier getUpdatedToken() {
+ return updatedToken;
+ }
+
+ /**
+ * Is this update a ResourceChange.
+ *
+ * @return isResourceChange.
+ */
+ public boolean isResourceChange() {
+ return isResourceChange;
+ }
+
+ /**
+ * Is this update an ExecType Update.
+ *
+ * @return isExecTypeUpdate.
+ */
+ public boolean isExecTypeUpdate() {
+ return isExecTypeUpdate;
+ }
+
+ /**
+ * Is this a container Increase.
+ *
+ * @return isIncrease.
+ */
+ public boolean isIncrease() {
+ return isIncrease;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 830a06d..e436822 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor
.ChangeMonitoringContainerResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
@@ -151,7 +152,9 @@ public class ContainerScheduler extends AbstractService implements
case SCHEDULE_CONTAINER:
scheduleContainer(event.getContainer());
break;
+ // NOTE: Is sent only after container state has changed to PAUSED...
case CONTAINER_PAUSED:
+ // NOTE: Is sent only after container state has changed to DONE...
case CONTAINER_COMPLETED:
onResourcesReclaimed(event.getContainer());
break;
@@ -180,58 +183,38 @@ public class ContainerScheduler extends AbstractService implements
if (updateEvent.isResourceChange()) {
if (runningContainers.containsKey(containerId)) {
this.utilizationTracker.subtractContainerResource(
- updateEvent.getContainer());
- updateEvent.getContainer().setContainerTokenIdentifier(
- updateEvent.getUpdatedToken());
+ new ContainerImpl(getConfig(), null, null, null, null,
+ updateEvent.getOriginalToken(), context));
this.utilizationTracker.addContainerResources(
updateEvent.getContainer());
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(containerId,
updateEvent.getUpdatedToken().getResource()));
- } else {
- // Is Queued or localizing..
- updateEvent.getContainer().setContainerTokenIdentifier(
- updateEvent.getUpdatedToken());
- }
- try {
- // Persist change in the state store.
- this.context.getNMStateStore().storeContainerResourceChanged(
- containerId,
- updateEvent.getUpdatedToken().getVersion(),
- updateEvent.getUpdatedToken().getResource());
- } catch (IOException e) {
- LOG.warn("Could not store container [" + containerId + "] resource " +
- "change..", e);
}
}
if (updateEvent.isExecTypeUpdate()) {
- updateEvent.getContainer().setContainerTokenIdentifier(
- updateEvent.getUpdatedToken());
- // If this is a running container.. just change the execution type
- // and be done with it.
- if (!runningContainers.containsKey(containerId)) {
- // Promotion or not (Increase signifies either a promotion
- // or container size increase)
- if (updateEvent.isIncrease()) {
- // Promotion of queued container..
- if (queuedOpportunisticContainers.remove(containerId) != null) {
- queuedGuaranteedContainers.put(containerId,
- updateEvent.getContainer());
- }
+ // Promotion or not (Increase signifies either a promotion
+ // or container size increase)
+ if (updateEvent.isIncrease()) {
+ // Promotion of queued container..
+ if (queuedOpportunisticContainers.remove(containerId) != null) {
+ queuedGuaranteedContainers.put(containerId,
+ updateEvent.getContainer());
//Kill/pause opportunistic containers if any to make room for
// promotion request
reclaimOpportunisticContainerResources(updateEvent.getContainer());
- } else {
- // Demotion of queued container.. Should not happen too often
- // since you should not find too many queued guaranteed
- // containers
- if (queuedGuaranteedContainers.remove(containerId) != null) {
- queuedOpportunisticContainers.put(containerId,
- updateEvent.getContainer());
- }
+ }
+ } else {
+ // Demotion of queued container.. Should not happen too often
+ // since you should not find too many queued guaranteed
+ // containers
+ if (queuedGuaranteedContainers.remove(containerId) != null) {
+ queuedOpportunisticContainers.put(containerId,
+ updateEvent.getContainer());
}
}
+ startPendingContainers(maxOppQueueLength <= 0);
}
}
@@ -290,6 +273,16 @@ public class ContainerScheduler extends AbstractService implements
queuedGuaranteedContainers.remove(container.getContainerId());
}
+ // Requeue PAUSED containers
+ if (container.getContainerState() == ContainerState.PAUSED) {
+ if (container.getContainerTokenIdentifier().getExecutionType() ==
+ ExecutionType.GUARANTEED) {
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
+ } else {
+ queuedOpportunisticContainers.put(
+ container.getContainerId(), container);
+ }
+ }
// decrement only if it was a running container
Container completedContainer = runningContainers.remove(container
.getContainerId());
@@ -301,7 +294,8 @@ public class ContainerScheduler extends AbstractService implements
ExecutionType.OPPORTUNISTIC) {
this.metrics.completeOpportunisticContainer(container.getResource());
}
- startPendingContainers(false);
+ boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
+ startPendingContainers(forceStartGuaranteedContainers);
}
}
@@ -311,26 +305,9 @@ public class ContainerScheduler extends AbstractService implements
* container without looking at available resource
*/
private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
- // Start pending guaranteed containers, if resources available.
+ // Start guaranteed containers that are paused, if resources available.
boolean resourcesAvailable = startContainers(
- queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
- // Resume opportunistic containers, if resource available.
- if (resourcesAvailable) {
- List<Container> pausedContainers = new ArrayList<Container>();
- Map<ContainerId, Container> containers =
- context.getContainers();
- for (Map.Entry<ContainerId, Container>entry : containers.entrySet()) {
- ContainerId contId = entry.getKey();
- // Find containers that were not already started and are in paused state
- if(false == runningContainers.containsKey(contId)) {
- if(containers.get(contId).getContainerState()
- == ContainerState.PAUSED) {
- pausedContainers.add(containers.get(contId));
- }
- }
- }
- resourcesAvailable = startContainers(pausedContainers, false);
- }
+ queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
// Start opportunistic containers, if resources available.
if (resourcesAvailable) {
startContainers(queuedOpportunisticContainers.values(), false);
@@ -590,16 +567,19 @@ public class ContainerScheduler extends AbstractService implements
queuedOpportunisticContainers.values().iterator();
while (containerIter.hasNext()) {
Container container = containerIter.next();
- if (numAllowed <= 0) {
- container.sendKillEvent(
- ContainerExitStatus.KILLED_BY_CONTAINER_SCHEDULER,
- "Container De-queued to meet NM queuing limits.");
- containerIter.remove();
- LOG.info(
- "Opportunistic container {} will be killed to meet NM queuing" +
- " limits.", container.getContainerId());
+ // Do not shed PAUSED containers
+ if (container.getContainerState() != ContainerState.PAUSED) {
+ if (numAllowed <= 0) {
+ container.sendKillEvent(
+ ContainerExitStatus.KILLED_BY_CONTAINER_SCHEDULER,
+ "Container De-queued to meet NM queuing limits.");
+ containerIter.remove();
+ LOG.info(
+ "Opportunistic container {} will be killed to meet NM queuing" +
+ " limits.", container.getContainerId());
+ }
+ numAllowed--;
}
- numAllowed--;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UpdateContainerSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UpdateContainerSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UpdateContainerSchedulerEvent.java
index 5384b7e..2473982 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UpdateContainerSchedulerEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UpdateContainerSchedulerEvent.java
@@ -21,33 +21,37 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
+
/**
* Update Event consumed by the {@link ContainerScheduler}.
*/
public class UpdateContainerSchedulerEvent extends ContainerSchedulerEvent {
- private ContainerTokenIdentifier updatedToken;
- private boolean isResourceChange;
- private boolean isExecTypeUpdate;
- private boolean isIncrease;
+ private final UpdateContainerTokenEvent containerEvent;
+ private final ContainerTokenIdentifier originalToken;
/**
* Create instance of Event.
*
- * @param originalContainer Original Container.
- * @param updatedToken Updated Container Token.
- * @param isResourceChange is this a Resource Change.
- * @param isExecTypeUpdate is this an ExecTypeUpdate.
- * @param isIncrease is this a Container Increase.
+ * @param container Container.
+ * @param origToken The Original Container Token.
+ * @param event The Container Event.
+ */
+ public UpdateContainerSchedulerEvent(Container container,
+ ContainerTokenIdentifier origToken, UpdateContainerTokenEvent event) {
+ super(container, ContainerSchedulerEventType.UPDATE_CONTAINER);
+ this.containerEvent = event;
+ this.originalToken = origToken;
+ }
+
+ /**
+ * Original Token before update.
+ *
+ * @return Container Token.
*/
- public UpdateContainerSchedulerEvent(Container originalContainer,
- ContainerTokenIdentifier updatedToken, boolean isResourceChange,
- boolean isExecTypeUpdate, boolean isIncrease) {
- super(originalContainer, ContainerSchedulerEventType.UPDATE_CONTAINER);
- this.updatedToken = updatedToken;
- this.isResourceChange = isResourceChange;
- this.isExecTypeUpdate = isExecTypeUpdate;
- this.isIncrease = isIncrease;
+ public ContainerTokenIdentifier getOriginalToken() {
+ return this.originalToken;
}
/**
@@ -56,7 +60,7 @@ public class UpdateContainerSchedulerEvent extends ContainerSchedulerEvent {
* @return Container Token.
*/
public ContainerTokenIdentifier getUpdatedToken() {
- return updatedToken;
+ return containerEvent.getUpdatedToken();
}
/**
@@ -64,7 +68,7 @@ public class UpdateContainerSchedulerEvent extends ContainerSchedulerEvent {
* @return isResourceChange.
*/
public boolean isResourceChange() {
- return isResourceChange;
+ return containerEvent.isResourceChange();
}
/**
@@ -72,7 +76,7 @@ public class UpdateContainerSchedulerEvent extends ContainerSchedulerEvent {
* @return isExecTypeUpdate.
*/
public boolean isExecTypeUpdate() {
- return isExecTypeUpdate;
+ return containerEvent.isExecTypeUpdate();
}
/**
@@ -80,6 +84,6 @@ public class UpdateContainerSchedulerEvent extends ContainerSchedulerEvent {
* @return isIncrease.
*/
public boolean isIncrease() {
- return isIncrease;
+ return containerEvent.isIncrease();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 3cafcbd..fc9e6c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -208,6 +208,8 @@ public abstract class BaseContainerManagerTest {
containerManager.init(conf);
nodeStatusUpdater.start();
((NMContext)context).setNodeStatusUpdater(nodeStatusUpdater);
+ ((NMContext)context).setContainerStateTransitionListener(
+ new NodeManager.DefaultContainerStateListener());
}
protected ContainerManagerImpl
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 38df208..6e8c005 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -90,12 +90,16 @@ import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrSignalContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestAuxServices.ServiceA;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
@@ -119,6 +123,41 @@ public class TestContainerManager extends BaseContainerManagerTest {
LOG = LoggerFactory.getLogger(TestContainerManager.class);
}
+ private static class Listener implements ContainerStateTransitionListener {
+
+ private final Map<ContainerId,
+ List<org.apache.hadoop.yarn.server.nodemanager.containermanager.
+ container.ContainerState>> states = new HashMap<>();
+ private final Map<ContainerId, List<ContainerEventType>> events =
+ new HashMap<>();
+
+ @Override
+ public void init(Context context) {}
+
+ @Override
+ public void preTransition(ContainerImpl op,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState beforeState,
+ ContainerEvent eventToBeProcessed) {
+ if (!states.containsKey(op.getContainerId())) {
+ states.put(op.getContainerId(), new ArrayList<>());
+ states.get(op.getContainerId()).add(beforeState);
+ events.put(op.getContainerId(), new ArrayList<>());
+ }
+ }
+
+ @Override
+ public void postTransition(ContainerImpl op,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState beforeState,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState afterState,
+ ContainerEvent processedEvent) {
+ states.get(op.getContainerId()).add(afterState);
+ events.get(op.getContainerId()).add(processedEvent.getType());
+ }
+ }
+
private boolean delayContainers = false;
@Override
@@ -144,7 +183,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
@Override
protected ContainerManagerImpl
createContainerManager(DeletionService delSrvc) {
- return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+ return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
metrics, dirsHandler) {
@Override
@@ -496,6 +535,9 @@ public class TestContainerManager extends BaseContainerManagerTest {
@Test
public void testContainerUpgradeSuccessAutoCommit() throws IOException,
InterruptedException, YarnException {
+ Listener listener = new Listener();
+ ((NodeManager.DefaultContainerStateListener)containerManager.context.
+ getContainerStateTransitionListener()).addListener(listener);
testContainerReInitSuccess(true);
// Should not be able to Commit (since already auto committed)
try {
@@ -504,6 +546,41 @@ public class TestContainerManager extends BaseContainerManagerTest {
} catch (Exception e) {
Assert.assertTrue(e.getMessage().contains("Nothing to Commit"));
}
+
+ List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState> containerStates =
+ listener.states.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.NEW,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.LOCALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING), containerStates);
+
+ List<ContainerEventType> containerEventTypes =
+ listener.events.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ ContainerEventType.INIT_CONTAINER,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.REINITIALIZE_CONTAINER,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ ContainerEventType.CONTAINER_LAUNCHED), containerEventTypes);
}
@Test
@@ -524,6 +601,9 @@ public class TestContainerManager extends BaseContainerManagerTest {
@Test
public void testContainerUpgradeSuccessExplicitRollback() throws IOException,
InterruptedException, YarnException {
+ Listener listener = new Listener();
+ ((NodeManager.DefaultContainerStateListener)containerManager.context.
+ getContainerStateTransitionListener()).addListener(listener);
String[] pids = testContainerReInitSuccess(false);
// Test that the container can be Restarted after the successful upgrrade.
@@ -575,6 +655,67 @@ public class TestContainerManager extends BaseContainerManagerTest {
Assert.assertNotEquals("The Rolled-back process should be a different pid",
pids[0], rolledBackPid);
+
+ List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState> containerStates =
+ listener.states.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.NEW,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.LOCALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ // This is the successful restart
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ // This is the rollback
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING), containerStates);
+
+ List<ContainerEventType> containerEventTypes =
+ listener.events.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ ContainerEventType.INIT_CONTAINER,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.REINITIALIZE_CONTAINER,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.REINITIALIZE_CONTAINER,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.ROLLBACK_REINIT,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ ContainerEventType.CONTAINER_LAUNCHED), containerEventTypes);
}
@Test
@@ -584,6 +725,9 @@ public class TestContainerManager extends BaseContainerManagerTest {
return;
}
containerManager.start();
+ Listener listener = new Listener();
+ ((NodeManager.DefaultContainerStateListener)containerManager.context.
+ getContainerStateTransitionListener()).addListener(listener);
// ////// Construct the Container-id
ContainerId cId = createContainerId(0);
File oldStartFile = new File(tmpDir, "start_file_o.txt").getAbsoluteFile();
@@ -598,6 +742,32 @@ public class TestContainerManager extends BaseContainerManagerTest {
// since upgrade was terminated..
Assert.assertTrue("Process is NOT alive!",
DefaultContainerExecutor.containerIsAlive(pid));
+
+ List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState> containerStates =
+ listener.states.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.NEW,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.LOCALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING), containerStates);
+
+ List<ContainerEventType> containerEventTypes =
+ listener.events.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ ContainerEventType.INIT_CONTAINER,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.REINITIALIZE_CONTAINER,
+ ContainerEventType.RESOURCE_FAILED), containerEventTypes);
}
@Test
@@ -632,6 +802,9 @@ public class TestContainerManager extends BaseContainerManagerTest {
return;
}
containerManager.start();
+ Listener listener = new Listener();
+ ((NodeManager.DefaultContainerStateListener)containerManager.context.
+ getContainerStateTransitionListener()).addListener(listener);
// ////// Construct the Container-id
ContainerId cId = createContainerId(0);
File oldStartFile = new File(tmpDir, "start_file_o.txt").getAbsoluteFile();
@@ -666,6 +839,50 @@ public class TestContainerManager extends BaseContainerManagerTest {
Assert.assertNotEquals("The Rolled-back process should be a different pid",
pid, rolledBackPid);
+
+ List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState> containerStates =
+ listener.states.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.NEW,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.LOCALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.REINITIALIZING_AWAITING_KILL,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING), containerStates);
+
+ List<ContainerEventType> containerEventTypes =
+ listener.events.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ ContainerEventType.INIT_CONTAINER,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.REINITIALIZE_CONTAINER,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_LAUNCHED), containerEventTypes);
}
/**
@@ -1582,16 +1799,12 @@ public class TestContainerManager extends BaseContainerManagerTest {
containerManager.updateContainer(updateRequest);
// Check response
Assert.assertEquals(
- 0, updateResponse.getSuccessfullyUpdatedContainers().size());
- Assert.assertEquals(2, updateResponse.getFailedRequests().size());
+ 1, updateResponse.getSuccessfullyUpdatedContainers().size());
+ Assert.assertEquals(1, updateResponse.getFailedRequests().size());
for (Map.Entry<ContainerId, SerializedException> entry : updateResponse
.getFailedRequests().entrySet()) {
Assert.assertNotNull("Failed message", entry.getValue().getMessage());
- if (cId0.equals(entry.getKey())) {
- Assert.assertTrue(entry.getValue().getMessage()
- .contains("Resource can only be changed when a "
- + "container is in RUNNING or SCHEDULED state"));
- } else if (cId7.equals(entry.getKey())) {
+ if (cId7.equals(entry.getKey())) {
Assert.assertTrue(entry.getValue().getMessage()
.contains("Container " + cId7.toString()
+ " is not handled by this NodeManager"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 7c74049..4b380ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -47,11 +48,17 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
@@ -76,6 +83,40 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
}
+ private static class Listener implements ContainerStateTransitionListener {
+
+ private final Map<ContainerId,
+ List<ContainerState>> states = new HashMap<>();
+ private final Map<ContainerId, List<ContainerEventType>> events =
+ new HashMap<>();
+
+ @Override
+ public void init(Context context) {}
+
+ @Override
+ public void preTransition(ContainerImpl op,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState beforeState,
+ ContainerEvent eventToBeProcessed) {
+ if (!states.containsKey(op.getContainerId())) {
+ states.put(op.getContainerId(), new ArrayList<>());
+ states.get(op.getContainerId()).add(beforeState);
+ events.put(op.getContainerId(), new ArrayList<>());
+ }
+ }
+
+ @Override
+ public void postTransition(ContainerImpl op,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState beforeState,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState afterState,
+ ContainerEvent processedEvent) {
+ states.get(op.getContainerId()).add(afterState);
+ events.get(op.getContainerId()).add(processedEvent.getType());
+ }
+ }
+
private boolean delayContainers = true;
@Override
@@ -542,6 +583,10 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
containerManager.start();
containerManager.getContainerScheduler().
setUsePauseEventForPreemption(true);
+
+ Listener listener = new Listener();
+ ((NodeManager.DefaultContainerStateListener)containerManager.getContext().
+ getContainerStateTransitionListener()).addListener(listener);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
@@ -606,6 +651,39 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
// starts running
BaseContainerManagerTest.waitForNMContainerState(containerManager,
createContainerId(0), ContainerState.DONE, 40);
+
+ List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState> containerStates =
+ listener.states.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.NEW,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.PAUSING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.PAUSED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RESUMING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.EXITED_WITH_SUCCESS,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.DONE), containerStates);
+ List<ContainerEventType> containerEventTypes =
+ listener.events.get(createContainerId(0));
+ Assert.assertEquals(Arrays.asList(ContainerEventType.INIT_CONTAINER,
+ ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.PAUSE_CONTAINER,
+ ContainerEventType.CONTAINER_PAUSED,
+ ContainerEventType.RESUME_CONTAINER,
+ ContainerEventType.CONTAINER_RESUMED,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP), containerEventTypes);
}
/**
@@ -1068,6 +1146,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
@Test
public void testPromotionOfOpportunisticContainers() throws Exception {
containerManager.start();
+ Listener listener = new Listener();
+ ((NodeManager.DefaultContainerStateListener)containerManager.getContext().
+ getContainerStateTransitionListener()).addListener(listener);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
@@ -1150,6 +1231,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
containerStatuses = containerManager
.getContainerStatuses(statRequest).getContainerStatuses();
Assert.assertEquals(1, containerStatuses.size());
+
for (ContainerStatus status : containerStatuses) {
if (org.apache.hadoop.yarn.api.records.ContainerState.RUNNING ==
status.getState()) {
@@ -1160,6 +1242,25 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
// Ensure no containers are queued.
Assert.assertEquals(0, containerScheduler.getNumQueuedContainers());
+
+ List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState> containerStates =
+ listener.states.get(createContainerId(1));
+ Assert.assertEquals(Arrays.asList(
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.NEW,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
+ ContainerState.RUNNING), containerStates);
+ List<ContainerEventType> containerEventTypes =
+ listener.events.get(createContainerId(1));
+ Assert.assertEquals(Arrays.asList(
+ ContainerEventType.INIT_CONTAINER,
+ ContainerEventType.UPDATE_CONTAINER_TOKEN,
+ ContainerEventType.CONTAINER_LAUNCHED), containerEventTypes);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HADOOP-14752.
TestCopyFromLocal#testCopyFromLocalWithThreads is fleaky. Contributed by
Andras Bokor.
Posted by kk...@apache.org.
HADOOP-14752. TestCopyFromLocal#testCopyFromLocalWithThreads is fleaky. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/442ea855
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/442ea855
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/442ea855
Branch: refs/heads/YARN-6592
Commit: 442ea85545ce3591f45211773a10346340abaed5
Parents: ceca969
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Oct 2 13:11:47 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Oct 2 13:11:47 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/shell/TestCopyFromLocal.java | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/442ea855/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
index 8d354b4..8e60540 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
@@ -115,10 +115,10 @@ public class TestCopyFromLocal {
Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
int numFiles = TestCopyFromLocal.initialize(dir);
int maxThreads = Runtime.getRuntime().availableProcessors() * 2;
- int randThreads = RandomUtils.nextInt(maxThreads);
- int numActualThreads = randThreads == 0 ? 1 : randThreads;
- String numThreads = Integer.toString(numActualThreads);
- run(new TestMultiThreadedCopy(numActualThreads, numFiles), "-t", numThreads,
+ int randThreads = RandomUtils.nextInt(maxThreads - 1) + 1;
+ String numThreads = Integer.toString(randThreads);
+ run(new TestMultiThreadedCopy(randThreads,
+ randThreads == 1 ? 0 : numFiles), "-t", numThreads,
new Path(dir, FROM_DIR_NAME).toString(),
new Path(dir, TO_DIR_NAME).toString());
}
@@ -144,6 +144,7 @@ public class TestCopyFromLocal {
}
private class TestMultiThreadedCopy extends CopyFromLocal {
+ public static final String NAME = "testCopyFromLocal";
private int expectedThreads;
private int expectedCompletedTaskCount;
@@ -164,9 +165,9 @@ public class TestCopyFromLocal {
// 2) There are no active tasks in the executor
// 3) Executor has shutdown correctly
ThreadPoolExecutor executor = getExecutor();
- Assert.assertEquals(executor.getCompletedTaskCount(),
- expectedCompletedTaskCount);
- Assert.assertEquals(executor.getActiveCount(), 0);
+ Assert.assertEquals(expectedCompletedTaskCount,
+ executor.getCompletedTaskCount());
+ Assert.assertEquals(0, executor.getActiveCount());
Assert.assertTrue(executor.isTerminated());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-6333. Improve doc for
minSharePreemptionTimeout,
fairSharePreemptionTimeout and fairSharePreemptionThreshold. (Contributed by
Chetna Chaudhari via Yufei Gu)
Posted by kk...@apache.org.
YARN-6333. Improve doc for minSharePreemptionTimeout, fairSharePreemptionTimeout and fairSharePreemptionThreshold. (Contributed by Chetna Chaudhari via Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66c41716
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66c41716
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66c41716
Branch: refs/heads/YARN-6592
Commit: 66c417167ae0d24467360ae3b222a10f00fa2700
Parents: ec2ae30
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Sep 29 18:07:33 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Sep 29 18:07:33 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/FairScheduler.md | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66c41716/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 748fda9..2ccd5ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -101,11 +101,11 @@ The allocation file must be in XML format. The format contains five types of ele
* **aclAdministerApps**: a list of users and/or groups that can administer a queue. Currently the only administrative action is killing an application. Refer to the ACLs section below for more info on the format of this list and how queue ACLs work.
- * **minSharePreemptionTimeout**: number of seconds the queue is under its minimum share before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue.
+ * **minSharePreemptionTimeout**: number of seconds the queue is under its minimum share before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is Long.MAX_VALUE, which means that it will not preempt containers until you set a meaningful value.
- * **fairSharePreemptionTimeout**: number of seconds the queue is under its fair share threshold before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue.
+ * **fairSharePreemptionTimeout**: number of seconds the queue is under its fair share threshold before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is Long.MAX_VALUE, which means that it will not preempt containers until you set a meaningful value.
- * **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue.
+ * **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is 0.5f.
* **allowPreemptionFrom**: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues.
@@ -113,11 +113,11 @@ The allocation file must be in XML format. The format contains five types of ele
* **A userMaxAppsDefault element**: which sets the default running app limit for any users whose limit is not otherwise specified.
-* **A defaultFairSharePreemptionTimeout element**: which sets the fair share preemption timeout for the root queue; overridden by fairSharePreemptionTimeout element in root queue.
+* **A defaultFairSharePreemptionTimeout element**: which sets the fair share preemption timeout for the root queue; overridden by fairSharePreemptionTimeout element in root queue. Default is set to Long.MAX_VALUE.
-* **A defaultMinSharePreemptionTimeout element**: which sets the min share preemption timeout for the root queue; overridden by minSharePreemptionTimeout element in root queue.
+* **A defaultMinSharePreemptionTimeout element**: which sets the min share preemption timeout for the root queue; overridden by minSharePreemptionTimeout element in root queue. Default is set to Long.MAX_VALUE.
-* **A defaultFairSharePreemptionThreshold element**: which sets the fair share preemption threshold for the root queue; overridden by fairSharePreemptionThreshold element in root queue.
+* **A defaultFairSharePreemptionThreshold element**: which sets the fair share preemption threshold for the root queue; overridden by fairSharePreemptionThreshold element in root queue. Default is set to 0.5f.
* **A queueMaxAppsDefault element**: which sets the default running app limit for queues; overridden by maxRunningApps element in each queue.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDFS-5040.Audit log for admin
commands/ logging output of all DFS admin commands. Contributed by Kuhu
Shukla.
Posted by kk...@apache.org.
HDFS-5040.Audit log for admin commands/ logging output of all DFS admin commands. Contributed by Kuhu Shukla.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d3e4ccc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d3e4ccc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d3e4ccc
Branch: refs/heads/YARN-6592
Commit: 9d3e4cccf9cd0ffb60ee0e7c65cea5ae3c8015c2
Parents: 14fec04
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Tue Sep 26 21:59:22 2017 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Tue Sep 26 21:59:22 2017 +0530
----------------------------------------------------------------------
.../hdfs/server/namenode/FSNamesystem.java | 149 ++--
.../hdfs/server/namenode/NameNodeRpcServer.java | 26 +-
.../hdfs/server/namenode/TestAuditLogger.java | 21 +-
.../namenode/TestAuditLoggerWithCommands.java | 692 ++++++++++++++++++-
.../server/namenode/TestNameNodeMXBean.java | 9 +-
5 files changed, 815 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d3e4ccc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 217df94..1534850 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -87,6 +87,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROU
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
@@ -351,7 +352,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
&& !auditLoggers.isEmpty();
}
- private void logAuditEvent(boolean succeeded, String cmd, String src)
+ void logAuditEvent(boolean succeeded, String cmd, String src)
throws IOException {
logAuditEvent(succeeded, cmd, src, null, null);
}
@@ -1715,7 +1716,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* Dump all metadata into specified file
*/
void metaSave(String filename) throws IOException {
- checkSuperuserPrivilege();
+ String operationName = "metaSave";
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
writeLock();
try {
@@ -1727,8 +1729,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
out.flush();
out.close();
} finally {
- writeUnlock("metaSave");
+ writeUnlock(operationName);
}
+ logAuditEvent(true, operationName, null);
}
private void metaSave(PrintWriter out) {
@@ -3123,7 +3126,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
requireEffectiveLayoutVersionForFeature(Feature.QUOTA_BY_STORAGE_TYPE);
}
checkOperation(OperationCategory.WRITE);
- final String operationName = "setQuota";
+ final String operationName = getQuotaCommand(nsQuota, ssQuota);
writeLock();
boolean success = false;
try {
@@ -4207,30 +4210,34 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
- DatanodeInfo[] datanodeReport(final DatanodeReportType type
- ) throws AccessControlException, StandbyException {
- checkSuperuserPrivilege();
+ DatanodeInfo[] datanodeReport(final DatanodeReportType type)
+ throws IOException {
+ String operationName = "datanodeReport";
+ DatanodeInfo[] arr;
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
readLock();
try {
checkOperation(OperationCategory.UNCHECKED);
final DatanodeManager dm = getBlockManager().getDatanodeManager();
final List<DatanodeDescriptor> results = dm.getDatanodeListForReport(type);
-
- DatanodeInfo[] arr = new DatanodeInfo[results.size()];
+ arr = new DatanodeInfo[results.size()];
for (int i=0; i<arr.length; i++) {
arr[i] = new DatanodeInfoBuilder().setFrom(results.get(i))
.build();
}
- return arr;
} finally {
- readUnlock("datanodeReport");
+ readUnlock(operationName);
}
+ logAuditEvent(true, operationName, null);
+ return arr;
}
DatanodeStorageReport[] getDatanodeStorageReport(final DatanodeReportType type
- ) throws AccessControlException, StandbyException {
- checkSuperuserPrivilege();
+ ) throws IOException {
+ String operationName = "getDatanodeStorageReport";
+ DatanodeStorageReport[] reports;
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
readLock();
try {
@@ -4238,17 +4245,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final DatanodeManager dm = getBlockManager().getDatanodeManager();
final List<DatanodeDescriptor> datanodes = dm.getDatanodeListForReport(type);
- DatanodeStorageReport[] reports = new DatanodeStorageReport[datanodes.size()];
+ reports = new DatanodeStorageReport[datanodes.size()];
for (int i = 0; i < reports.length; i++) {
final DatanodeDescriptor d = datanodes.get(i);
reports[i] = new DatanodeStorageReport(
new DatanodeInfoBuilder().setFrom(d).build(),
d.getStorageReports());
}
- return reports;
} finally {
readUnlock("getDatanodeStorageReport");
}
+ logAuditEvent(true, operationName, null);
+ return reports;
}
/**
@@ -4258,8 +4266,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
boolean saveNamespace(final long timeWindow, final long txGap)
throws IOException {
+ String operationName = "saveNamespace";
checkOperation(OperationCategory.UNCHECKED);
- checkSuperuserPrivilege();
+ checkSuperuserPrivilege(operationName);
boolean saved = false;
cpLock(); // Block if a checkpointing is in progress on standby.
@@ -4273,12 +4282,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
saved = getFSImage().saveNamespace(timeWindow, txGap, this);
} finally {
- readUnlock("saveNamespace");
+ readUnlock(operationName);
cpUnlock();
}
if (saved) {
LOG.info("New namespace image has been created");
}
+ logAuditEvent(true, operationName, null);
return saved;
}
@@ -4288,9 +4298,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*
* @throws AccessControlException if superuser privilege is violated.
*/
- boolean restoreFailedStorage(String arg) throws AccessControlException,
- StandbyException {
- checkSuperuserPrivilege();
+ boolean restoreFailedStorage(String arg) throws IOException {
+ String operationName = getFailedStorageCommand(arg);
+ boolean val = false;
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
cpLock(); // Block if a checkpointing is in progress on standby.
writeLock();
@@ -4298,17 +4309,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.UNCHECKED);
// if it is disabled - enable it and vice versa.
- if(arg.equals("check"))
- return getFSImage().getStorage().getRestoreFailedStorage();
-
- boolean val = arg.equals("true"); // false if not
- getFSImage().getStorage().setRestoreFailedStorage(val);
-
- return val;
+ if(arg.equals("check")) {
+ val = getFSImage().getStorage().getRestoreFailedStorage();
+ } else {
+ val = arg.equals("true"); // false if not
+ getFSImage().getStorage().setRestoreFailedStorage(val);
+ }
} finally {
- writeUnlock("restoreFailedStorage");
+ writeUnlock(operationName);
cpUnlock();
}
+ logAuditEvent(true, operationName, null);
+ return val;
}
Date getStartTime() {
@@ -4316,7 +4328,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
void finalizeUpgrade() throws IOException {
- checkSuperuserPrivilege();
+ String operationName = "finalizeUpgrade";
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.UNCHECKED);
cpLock(); // Block if a checkpointing is in progress on standby.
writeLock();
@@ -4324,26 +4337,33 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.UNCHECKED);
getFSImage().finalizeUpgrade(this.isHaEnabled() && inActiveState());
} finally {
- writeUnlock("finalizeUpgrade");
+ writeUnlock(operationName);
cpUnlock();
}
+ logAuditEvent(true, operationName, null);
}
void refreshNodes() throws IOException {
+ String operationName = "refreshNodes";
checkOperation(OperationCategory.UNCHECKED);
- checkSuperuserPrivilege();
+ checkSuperuserPrivilege(operationName);
getBlockManager().getDatanodeManager().refreshNodes(new HdfsConfiguration());
+ logAuditEvent(true, operationName, null);
}
void setBalancerBandwidth(long bandwidth) throws IOException {
+ String operationName = "setBalancerBandwidth";
checkOperation(OperationCategory.UNCHECKED);
- checkSuperuserPrivilege();
+ checkSuperuserPrivilege(operationName);
getBlockManager().getDatanodeManager().setBalancerBandwidth(bandwidth);
+ logAuditEvent(true, operationName, null);
}
boolean setSafeMode(SafeModeAction action) throws IOException {
+ String operationName = action.toString().toLowerCase();
+ boolean error = false;
if (action != SafeModeAction.SAFEMODE_GET) {
- checkSuperuserPrivilege();
+ checkSuperuserPrivilege(operationName);
switch(action) {
case SAFEMODE_LEAVE: // leave safe mode
leaveSafeMode(false);
@@ -4356,8 +4376,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
break;
default:
LOG.error("Unexpected safe mode action");
+ error = true;
}
}
+ if (!error) {
+ logAuditEvent(true, operationName, null);
+ }
return isInSafeMode();
}
@@ -4493,7 +4517,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
CheckpointSignature rollEditLog() throws IOException {
- checkSuperuserPrivilege();
+ String operationName = "rollEditLog";
+ CheckpointSignature result = null;
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.JOURNAL);
writeLock();
try {
@@ -4502,10 +4528,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (Server.isRpcInvocation()) {
LOG.info("Roll Edit Log from " + Server.getRemoteAddress());
}
- return getFSImage().rollEditLog(getEffectiveLayoutVersion());
+ result = getFSImage().rollEditLog(getEffectiveLayoutVersion());
} finally {
- writeUnlock("rollEditLog");
+ writeUnlock(operationName);
}
+ logAuditEvent(true, operationName, null);
+ return result;
}
NamenodeCommand startCheckpoint(NamenodeRegistration backupNode,
@@ -6194,11 +6222,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.WRITE);
final String operationName = "allowSnapshot";
boolean success = false;
+ checkSuperuserPrivilege(operationName);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot allow snapshot for " + path);
- checkSuperuserPrivilege();
FSDirSnapshotOp.allowSnapshot(dir, snapshotManager, path);
success = true;
} finally {
@@ -6212,12 +6240,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void disallowSnapshot(String path) throws IOException {
checkOperation(OperationCategory.WRITE);
final String operationName = "disallowSnapshot";
+ checkSuperuserPrivilege(operationName);
boolean success = false;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot disallow snapshot for " + path);
- checkSuperuserPrivilege();
FSDirSnapshotOp.disallowSnapshot(dir, snapshotManager, path);
success = true;
} finally {
@@ -6410,7 +6438,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
RollingUpgradeInfo queryRollingUpgrade() throws IOException {
- checkSuperuserPrivilege();
+ final String operationName = "queryRollingUpgrade";
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.READ);
readLock();
try {
@@ -6420,15 +6449,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
Preconditions.checkNotNull(rollingUpgradeInfo);
boolean hasRollbackImage = this.getFSImage().hasRollbackFSImage();
rollingUpgradeInfo.setCreatedRollbackImages(hasRollbackImage);
- return rollingUpgradeInfo;
} finally {
- readUnlock("queryRollingUpgrade");
+ readUnlock(operationName);
}
+ logAuditEvent(true, operationName, null, null, null);
+ return rollingUpgradeInfo;
}
RollingUpgradeInfo startRollingUpgrade() throws IOException {
final String operationName = "startRollingUpgrade";
- checkSuperuserPrivilege();
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -6619,7 +6649,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
RollingUpgradeInfo finalizeRollingUpgrade() throws IOException {
final String operationName = "finalizeRollingUpgrade";
- checkSuperuserPrivilege();
+ checkSuperuserPrivilege(operationName);
checkOperation(OperationCategory.WRITE);
writeLock();
try {
@@ -7744,5 +7774,38 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.size();
}
+ void checkSuperuserPrivilege(String operationName)
+ throws IOException {
+ try {
+ checkSuperuserPrivilege();
+ } catch (AccessControlException ace) {
+ logAuditEvent(false, operationName, null);
+ throw ace;
+ }
+ }
+
+ String getQuotaCommand(long nsQuota, long dsQuota) {
+ if (nsQuota == HdfsConstants.QUOTA_RESET
+ && dsQuota == HdfsConstants.QUOTA_DONT_SET) {
+ return "clearQuota";
+ } else if (nsQuota == HdfsConstants.QUOTA_DONT_SET
+ && dsQuota == HdfsConstants.QUOTA_RESET) {
+ return "clearSpaceQuota";
+ } else if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
+ return "setQuota";
+ } else {
+ return "setSpaceQuota";
+ }
+ }
+
+ String getFailedStorageCommand(String mode) {
+ if(mode.equals("check")) {
+ return "checkRestoreFailedStorage";
+ } else if (mode.equals("true")) {
+ return "enableRestoreFailedStorage";
+ } else {
+ return "disableRestoreFailedStorage";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d3e4ccc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index c07b521..2512b57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -1624,6 +1624,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
if (this.serviceRpcServer != null) {
this.serviceRpcServer.refreshServiceAcl(new Configuration(), new HDFSPolicyProvider());
}
+ namesystem.logAuditEvent(true, "refreshServiceAcl", null);
}
@Override // RefreshAuthorizationPolicyProtocol
@@ -1631,17 +1632,19 @@ public class NameNodeRpcServer implements NamenodeProtocols {
LOG.info("Refreshing all user-to-groups mappings. Requested by user: " +
getRemoteUser().getShortUserName());
Groups.getUserToGroupsMappingService().refresh();
+ namesystem.logAuditEvent(true, "refreshUserToGroupsMappings", null);
}
@Override // RefreshAuthorizationPolicyProtocol
- public void refreshSuperUserGroupsConfiguration() {
+ public void refreshSuperUserGroupsConfiguration() throws IOException {
LOG.info("Refreshing SuperUser proxy group mapping list ");
ProxyUsers.refreshSuperUserGroupsConfiguration();
+ namesystem.logAuditEvent(true, "refreshSuperUserGroupsConfiguration", null);
}
@Override // RefreshCallQueueProtocol
- public void refreshCallQueue() {
+ public void refreshCallQueue() throws IOException {
LOG.info("Refreshing call queue.");
Configuration conf = new Configuration();
@@ -1649,6 +1652,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
if (this.serviceRpcServer != null) {
serviceRpcServer.refreshCallQueue(conf);
}
+ namesystem.logAuditEvent(true, "refreshCallQueue", null);
}
@Override // GenericRefreshProtocol
@@ -2412,22 +2416,30 @@ public class NameNodeRpcServer implements NamenodeProtocols {
@Override // ReconfigurationProtocol
public void startReconfiguration() throws IOException {
checkNNStartup();
- namesystem.checkSuperuserPrivilege();
+ String operationName = "startNamenodeReconfiguration";
+ namesystem.checkSuperuserPrivilege(operationName);
nn.startReconfigurationTask();
+ namesystem.logAuditEvent(true, operationName, null);
}
@Override // ReconfigurationProtocol
public ReconfigurationTaskStatus getReconfigurationStatus()
throws IOException {
checkNNStartup();
- namesystem.checkSuperuserPrivilege();
- return nn.getReconfigurationTaskStatus();
+ String operationName = "getNamenodeReconfigurationStatus";
+ namesystem.checkSuperuserPrivilege(operationName);
+ ReconfigurationTaskStatus status = nn.getReconfigurationTaskStatus();
+ namesystem.logAuditEvent(true, operationName, null);
+ return status;
}
@Override // ReconfigurationProtocol
public List<String> listReconfigurableProperties() throws IOException {
checkNNStartup();
- namesystem.checkSuperuserPrivilege();
- return Lists.newArrayList(nn.getReconfigurableProperties());
+ String operationName = "listNamenodeReconfigurableProperties";
+ namesystem.checkSuperuserPrivilege(operationName);
+ List<String> result = Lists.newArrayList(nn.getReconfigurableProperties());
+ namesystem.logAuditEvent(true, operationName, null);
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d3e4ccc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
index 0e3cc8d..c422f32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
@@ -152,7 +152,6 @@ public class TestAuditLogger {
conf.set(DFS_NAMENODE_AUDIT_LOGGERS_KEY,
DummyAuditLogger.class.getName());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
-
GetOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
try {
cluster.waitClusterUp();
@@ -168,7 +167,8 @@ public class TestAuditLogger {
conn.connect();
assertEquals(200, conn.getResponseCode());
conn.disconnect();
- assertEquals(1, DummyAuditLogger.logCount);
+ assertEquals("getfileinfo", DummyAuditLogger.lastCommand);
+ DummyAuditLogger.resetLogCount();
assertEquals("127.0.0.1", DummyAuditLogger.remoteAddr);
// non-trusted proxied request
@@ -178,7 +178,9 @@ public class TestAuditLogger {
conn.connect();
assertEquals(200, conn.getResponseCode());
conn.disconnect();
- assertEquals(2, DummyAuditLogger.logCount);
+ assertEquals("getfileinfo", DummyAuditLogger.lastCommand);
+ assertTrue(DummyAuditLogger.logCount == 1);
+ DummyAuditLogger.resetLogCount();
assertEquals("127.0.0.1", DummyAuditLogger.remoteAddr);
// trusted proxied request
@@ -190,7 +192,8 @@ public class TestAuditLogger {
conn.connect();
assertEquals(200, conn.getResponseCode());
conn.disconnect();
- assertEquals(3, DummyAuditLogger.logCount);
+ assertEquals("getfileinfo", DummyAuditLogger.lastCommand);
+ assertTrue(DummyAuditLogger.logCount == 1);
assertEquals("1.1.1.1", DummyAuditLogger.remoteAddr);
} finally {
cluster.shutdown();
@@ -547,6 +550,7 @@ public class TestAuditLogger {
static int unsuccessfulCount;
static short foundPermission;
static String remoteAddr;
+ private static String lastCommand;
public void initialize(Configuration conf) {
initialized = true;
@@ -565,11 +569,16 @@ public class TestAuditLogger {
if (!succeeded) {
unsuccessfulCount++;
}
+ lastCommand = cmd;
if (stat != null) {
foundPermission = stat.getPermission().toShort();
}
}
+ public static String getLastCommand() {
+ return lastCommand;
+ }
+
}
public static class BrokenAuditLogger implements AuditLogger {
@@ -581,7 +590,9 @@ public class TestAuditLogger {
public void logAuditEvent(boolean succeeded, String userName,
InetAddress addr, String cmd, String src, String dst,
FileStatus stat) {
- throw new RuntimeException("uh oh");
+ if (!cmd.equals("datanodeReport")) {
+ throw new RuntimeException("uh oh");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d3e4ccc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
index 2adf470..4eda88f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BatchedRemoteIterator;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -31,24 +32,30 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
-import org.junit.BeforeClass;
-import org.junit.AfterClass;
-import org.junit.Ignore;
+import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import org.junit.Before;
import org.junit.Test;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import org.mockito.Mock;
import org.mockito.Mockito;
import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
public class TestAuditLoggerWithCommands {
@@ -65,13 +72,15 @@ public class TestAuditLoggerWithCommands {
static UserGroupInformation user2;
private static NamenodeProtocols proto;
- @BeforeClass
- public static void initialize() throws Exception {
+ @Before
+ public void initialize() throws Exception {
// start a cluster
conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
conf.setBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ conf.setBoolean(
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true);
cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
cluster.waitActive();
@@ -88,8 +97,9 @@ public class TestAuditLoggerWithCommands {
fs = cluster.getFileSystem();
}
- @AfterClass
- public static void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
+ Server.getCurCall().set(null);
fs.close();
fs2.close();
fileSys.close();
@@ -126,22 +136,29 @@ public class TestAuditLoggerWithCommands {
Path path = new Path("/testdir/testdir1");
fs.mkdirs(path);
fileSys = DFSTestUtil.getFileSystemAs(user1, conf);
- try {
- ((DistributedFileSystem)fileSys).setQuota(path, 10l, 10l);
- fail("The operation should have failed with AccessControlException");
- } catch (AccessControlException ace) {
- }
- String acePattern =
- ".*allowed=false.*ugi=theDoctor.*cmd=setQuota.*";
- int length = verifyAuditLogs(acePattern);
+ verifySetQuota(path, HdfsConstants.QUOTA_RESET,
+ HdfsConstants.QUOTA_DONT_SET);
+ verifySetQuota(path, HdfsConstants.QUOTA_DONT_SET,
+ HdfsConstants.QUOTA_RESET);
+ verifySetQuota(path, HdfsConstants.QUOTA_DONT_SET,
+ HdfsConstants.BYTES_IN_INTEGER);
+ verifySetQuota(path, HdfsConstants.BYTES_IN_INTEGER,
+ HdfsConstants.BYTES_IN_INTEGER);
fileSys.close();
+ }
+
+ private void verifySetQuota(Path path, long nsQuota, long ssQuota)
+ throws IOException {
+ String operationName = cluster.getNamesystem().getQuotaCommand(
+ nsQuota, ssQuota);
+ String acePattern =
+ ".*allowed=false.*ugi=theDoctor.*cmd=.*" + operationName + ".*";
try {
- ((DistributedFileSystem)fileSys).setQuota(path, 10l, 10l);
- fail("The operation should have failed with IOException");
- } catch (IOException ace) {
+ ((DistributedFileSystem) fileSys).setQuota(path, nsQuota, ssQuota);
+ fail("The operation should have failed");
+ } catch (AccessControlException ace) {
}
- assertTrue("Unexpected log from getContentSummary",
- length == auditlog.getOutput().split("\n").length);
+ verifyAuditLogs(acePattern);
}
@Test
@@ -180,7 +197,7 @@ public class TestAuditLoggerWithCommands {
".*allowed=false.*ugi=theDoctor.*cmd=renameSnapshot.*";
fs.mkdirs(srcDir);
fileSys = DFSTestUtil.getFileSystemAs(user1, conf);
- cluster.getNamesystem().allowSnapshot(srcDir.toString());
+ ((DistributedFileSystem)fs).allowSnapshot(srcDir);
try {
fileSys.createSnapshot(srcDir);
fail("The operation should have failed with AccessControlException");
@@ -215,7 +232,7 @@ public class TestAuditLoggerWithCommands {
Path s1;
fs.mkdirs(srcDir);
fileSys = DFSTestUtil.getFileSystemAs(user1, conf);
- cluster.getNamesystem().allowSnapshot(srcDir.toString());
+ ((DistributedFileSystem)fs).allowSnapshot(srcDir);
try {
s1 = fs.createSnapshot(srcDir);
fileSys.deleteSnapshot(srcDir, s1.getName());
@@ -237,12 +254,65 @@ public class TestAuditLoggerWithCommands {
}
@Test
+ public void testAllowSnapshot() throws Exception {
+ Path srcDir = new Path(System.getProperty("user.dir"), "/src");
+ fs.mkdirs(srcDir);
+ String pattern =
+ ".*allowed=true.*ugi=" +
+ System.getProperty("user.name")+".*cmd=allowSnapshot.*";
+ try {
+ ((DistributedFileSystem)fs).allowSnapshot(srcDir);
+ verifyAuditLogs(pattern);
+ } catch (Exception e) {
+ fail("The operation should not have failed with Exception");
+ }
+ fileSys = DFSTestUtil.getFileSystemAs(user1, conf);
+ try {
+ ((DistributedFileSystem)fileSys).allowSnapshot(srcDir);
+ fail("The operation should have failed with AccessControlException");
+ } catch (AccessControlException ace) {
+ }
+ pattern =
+ ".*allowed=false.*ugi=theDoctor.*cmd=allowSnapshot.*";
+ verifyAuditLogs(pattern);
+ fs.delete(srcDir, true);
+ fileSys.close();
+ }
+
+ @Test
+ public void testDisallowSnapshot() throws Exception {
+ Path srcDir = new Path(System.getProperty("user.dir"), "/src");
+ fs.mkdirs(srcDir);
+ cluster.getNamesystem().allowSnapshot(srcDir.toString());
+ String pattern =
+ ".*allowed=true.*ugi=" +
+ System.getProperty("user.name")+".*cmd=disallowSnapshot.*";
+ try {
+ ((DistributedFileSystem)fs).disallowSnapshot(srcDir);
+ verifyAuditLogs(pattern);
+ } catch (Exception e) {
+ fail("The operation should not have failed with Exception");
+ }
+ cluster.getNamesystem().allowSnapshot(srcDir.toString());
+ fileSys = DFSTestUtil.getFileSystemAs(user1, conf);
+ try {
+ ((DistributedFileSystem)fileSys).disallowSnapshot(srcDir);
+ fail("The operation should have failed with AccessControlException");
+ } catch (AccessControlException ace) {
+ pattern =
+ ".*allowed=false.*ugi=theDoctor.*cmd=disallowSnapshot.*";
+ verifyAuditLogs(pattern);
+ }
+ fileSys.close();
+ }
+
+ @Test
public void testAddCacheDirective() throws Exception {
removeExistingCachePools(null);
proto.addCachePool(new CachePoolInfo("pool1").
setMode(new FsPermission((short) 0)));
CacheDirectiveInfo alpha = new CacheDirectiveInfo.Builder().
- setPath(new Path("/alpha")).
+ setPath(new Path(System.getProperty("user.dir"), "/alpha")).
setPool("pool1").
build();
fileSys = DFSTestUtil.getFileSystemAs(user1, conf);
@@ -618,6 +688,579 @@ public class TestAuditLoggerWithCommands {
return verifyAuditLogs(".*allowed=" + allowed + pattern);
}
+ @Test
+ public void testMetaSave() throws Exception {
+ String aceMetaSave =
+ ".*allowed=true.*cmd=metaSave.*";
+ try {
+ ((DistributedFileSystem)fs).metaSave("test.log");
+ verifyAuditLogs(aceMetaSave);
+ } catch (Exception e) {
+ fail("The operation should not have failed with Exception");
+ }
+ try {
+ ((DistributedFileSystem)fileSys).metaSave("test.log");
+ fail("The operation should have failed with AccessControlException");
+ } catch (IOException ace) {
+ GenericTestUtils.assertExceptionContains("Access denied", ace);
+ aceMetaSave =
+ ".*allowed=false.*cmd=metaSave.*";
+ verifyAuditLogs(aceMetaSave);
+ }
+ }
+
+ @Test
+ public void testStartReconfiguration() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=startNamenodeReconfiguration.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ ((NameNodeRpcServer) cluster.getNameNodeRpc()).startReconfiguration();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("StartConfiguration should have passed!");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ ((NameNodeRpcServer)cluster.getNameNodeRpc()).startReconfiguration();
+ fail(
+ "startNameNodeReconfiguration should throw AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=startNamenodeReconfiguration.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testGetReconfigurationStatus() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=getNamenodeReconfigurationStatus.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ ((NameNodeRpcServer) cluster.getNameNodeRpc()).getReconfigurationStatus();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("getNamenodeReconfigurationStatus " +
+ " threw Exception!");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ ((NameNodeRpcServer)cluster.getNameNodeRpc()).getReconfigurationStatus();
+ fail("getNamenodeReconfigurationStatus " +
+ " did not throw AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=getNamenodeReconfigurationStatus.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testListReconfigurableProperties() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=listNamenodeReconfigurableProperties.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ ((NameNodeRpcServer) cluster.getNameNodeRpc()).
+ listReconfigurableProperties();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("listReconfigurableProperties " +
+ " threw Exception!");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ ((NameNodeRpcServer)cluster.getNameNodeRpc()).
+ listReconfigurableProperties();
+ fail("getNamenodeReconfigurationStatus " +
+ " did not throw AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=listNamenodeReconfigurableProperties.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testRefreshUserToGroupsMappings() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=refreshUserToGroupsMappings.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ ((NameNodeRpcServer)cluster.getNameNodeRpc()).
+ refreshUserToGroupsMappings();
+ verifyAuditLogs(auditLogString);
+ }
+
+ @Test
+ public void testRefreshSuperUserGroupsConfiguration() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=refreshSuperUserGroupsConfiguration.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ ((NameNodeRpcServer) cluster.getNameNodeRpc()).
+ refreshSuperUserGroupsConfiguration();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail(" The operation threw an exception");
+ }
+ }
+
+ @Test
+ public void testRefreshQueue() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=refreshCallQueue.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ ((NameNodeRpcServer) cluster.getNameNodeRpc()).refreshCallQueue();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail(" The operation threw an exception");
+ }
+ }
+
+ @Test
+ public void testRefreshServiceAcl() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=refreshServiceAcl.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ ((NameNodeRpcServer) cluster.getNameNodeRpc()).refreshServiceAcl();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail(" The operation threw an exception" + e);
+ }
+ }
+
+ @Test
+ public void testFinalizeRollingUpgrade() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=finalizeRollingUpgrade.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ fsNamesystem.setRollingUpgradeInfo(false, System.currentTimeMillis());
+ try {
+ fsNamesystem.finalizeRollingUpgrade();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("finalizeRollingUpgrade threw Exception");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.finalizeRollingUpgrade();
+ fail("finalizeRollingUpgrade should throw AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=finalizeRollingUpgrade.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testQueryRollingUpgrade() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=queryRollingUpgrade.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ fsNamesystem.setRollingUpgradeInfo(false, System.currentTimeMillis());
+ try {
+ fsNamesystem.queryRollingUpgrade();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("queryRollingUpgrade threw Exception");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.queryRollingUpgrade();
+ fail("queryRollingUpgrade should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=queryRollingUpgrade.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testRollEditLog() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=rollEditLog.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ fsNamesystem.rollEditLog();
+ } catch (Exception e) {
+ fail("rollEditLog threw Exception");
+ }
+ verifyAuditLogs(auditLogString);
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.rollEditLog();
+ fail("rollEditLog should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=rollEditLog.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testSetSafeMode() throws Exception {
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ verifySuccessfulSetSafeMode(fsNamesystem,
+ HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ verifySuccessfulSetSafeMode(fsNamesystem,
+ HdfsConstants.SafeModeAction.SAFEMODE_GET);
+ verifySuccessfulSetSafeMode(fsNamesystem,
+ HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ verifySuccessfulSetSafeMode(fsNamesystem,
+ HdfsConstants.SafeModeAction.SAFEMODE_FORCE_EXIT);
+ String auditLogString;
+ auditLogString =
+ ".*allowed=true.*cmd=safemode_get.*";
+ fsNamesystem.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET);
+ verifyAuditLogs(auditLogString);
+ auditLogString =
+ ".*allowed=true.*cmd=safemode_leave.*";
+ fsNamesystem.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ verifyAuditLogs(auditLogString);
+ auditLogString =
+ ".*allowed=true.*cmd=safemode_force_exit.*";
+ fsNamesystem.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_FORCE_EXIT);
+ verifyAuditLogs(auditLogString);
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ verifySafeModeAction(fsNamesystem,
+ HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ verifySafeModeAction(fsNamesystem,
+ HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ verifySafeModeAction(fsNamesystem,
+ HdfsConstants.SafeModeAction.SAFEMODE_FORCE_EXIT);
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ }
+
+ @Test
+ public void testSetBalancerBandwidth() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=setBalancerBandwidth.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ fsNamesystem.setBalancerBandwidth(10);
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("setBalancerBandwidth threw exception!");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.setBalancerBandwidth(10);
+ fail(
+ "setBalancerBandwidth should have thrown AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=setBalancerBandwidth.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testRefreshNodes() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=refreshNodes.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ fsNamesystem.refreshNodes();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("refreshNodes threw exception!");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.refreshNodes();
+ fail(
+ "refreshNodes should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=refreshNodes.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testFinalizeUpgrade() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=finalizeUpgrade.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ fsNamesystem.finalizeUpgrade();
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("finalizeUpgrade threw Exception");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.finalizeUpgrade();
+ fail("finalizeUpgrade should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=finalizeUpgrade.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testSaveNamespace() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=saveNamespace.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ fsNamesystem.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ try {
+ fsNamesystem.saveNamespace(10, 10);
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("saveNamespace threw Exception");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.saveNamespace(10, 10);
+ fail("saveNamespace should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=saveNamespace.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testDatanodeReport() throws Exception {
+ String auditLogString =
+ ".*allowed=true.*cmd=datanodeReport.*";
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ try {
+ fsNamesystem.datanodeReport(HdfsConstants.DatanodeReportType.ALL);
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("datanodeReport threw Exception");
+ }
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ try {
+ fsNamesystem.datanodeReport(HdfsConstants.DatanodeReportType.ALL);
+ fail(
+ "datanodeReport should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=datanodeReport.*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ @Test
+ public void testRestoreFailedStorage() throws Exception {
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ verifyAuditRestoreFailedStorage(fsNamesystem, "check");
+ verifyAuditRestoreFailedStorage(fsNamesystem, "true");
+ verifyAuditRestoreFailedStorage(fsNamesystem, "false");
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ verifyAuditRestoreFailedStorageACE(fsNamesystem, "check");
+ verifyAuditRestoreFailedStorageACE(fsNamesystem, "true");
+ verifyAuditRestoreFailedStorageACE(fsNamesystem, "false");
+ }
+
+ @Test
+ public void testGetDatanodeStorageReport() throws Exception {
+ FSNamesystem fsNamesystem = spy(cluster.getNamesystem());
+ when(fsNamesystem.isExternalInvocation()).thenReturn(true);
+ Server.Call call = spy(new Server.Call(
+ 1, 1, null, null, RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3}));
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser(System.getProperty("user.name")));
+ Server.getCurCall().set(call);
+ DatanodeStorageReport[] reports = fsNamesystem.getDatanodeStorageReport(
+ HdfsConstants.DatanodeReportType.ALL);
+ String auditLogString =
+ ".*allowed=true.*cmd=" + "getDatanodeStorageReport" + ".*";
+ verifyAuditLogs(auditLogString);
+ when(call.getRemoteUser()).thenReturn(
+ UserGroupInformation.createRemoteUser("theDoctor"));
+ auditLogString =
+ ".*allowed=false.*cmd=" + "getDatanodeStorageReport" + ".*";
+ try {
+ fsNamesystem.getDatanodeStorageReport(
+ HdfsConstants.DatanodeReportType.ALL);
+ fail("Should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ private void verifyAuditRestoreFailedStorageACE(
+ FSNamesystem fsNamesystem, String arg) throws IOException {
+ String operationName = fsNamesystem.getFailedStorageCommand(arg);
+ try {
+ fsNamesystem.restoreFailedStorage(arg);
+ fail(
+ "RestoreFailedStorage should have thrown AccessControlException!");
+ } catch (IOException ace) {
+ assertEquals("Unexpected Exception!",
+ ace.getClass(), AccessControlException.class);
+ String auditLogString =
+ ".*allowed=false.*cmd=" + operationName + ".*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
+ private void verifyAuditRestoreFailedStorage(
+ FSNamesystem fsNamesystem, String arg) throws IOException {
+ String operationName = fsNamesystem.getFailedStorageCommand(arg);
+ String auditLogString =
+ ".*allowed=true.*cmd=" + operationName + ".*";
+ try {
+ fsNamesystem.restoreFailedStorage(arg);
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail(
+ "The operation should not have failed with Exception");
+ }
+ }
+
+ private void verifySuccessfulSetSafeMode(FSNamesystem fsNamesystem,
+ HdfsConstants.SafeModeAction safeModeAction) throws IOException {
+ String operationName = safeModeAction.toString().toLowerCase();
+ String auditLogString =
+ ".*allowed=true.*cmd=" + operationName +".*";
+ try {
+ fsNamesystem.setSafeMode(safeModeAction);
+ verifyAuditLogs(auditLogString);
+ } catch (Exception e) {
+ fail("The operation should not have failed with Exception");
+ }
+ }
+
+ private void verifySafeModeAction(
+ FSNamesystem fsNamesystem, HdfsConstants.SafeModeAction safeModeAction)
+ throws IOException {
+ String operationName = safeModeAction.toString().toLowerCase();
+ String auditLogString;
+ try {
+ fsNamesystem.setSafeMode(safeModeAction);
+ fail("setSafeMode should have thrown an AccessControlException!");
+ } catch (AccessControlException ace) {
+ auditLogString =
+ ".*allowed=false.*cmd=" + operationName +".*";
+ verifyAuditLogs(auditLogString);
+ }
+ }
+
private int verifyAuditLogs(String pattern) {
int length = auditlog.getOutput().split("\n").length;
String lastAudit = auditlog.getOutput().split("\n")[length - 1];
@@ -633,4 +1276,3 @@ public class TestAuditLoggerWithCommands {
}
}
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d3e4ccc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 36638e0..93a7525 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -556,13 +556,18 @@ public class TestNameNodeMXBean {
assertEquals("Unexpected num windows", 3, windows.size());
for (Map<String, List<Map<String, Object>>> window : windows) {
final List<Map<String, Object>> ops = window.get("ops");
- assertEquals("Unexpected num ops", 3, ops.size());
+ assertEquals("Unexpected num ops", 4, ops.size());
for (Map<String, Object> op: ops) {
+ if (op.get("opType").equals("datanodeReport")) {
+ continue;
+ }
final long count = Long.parseLong(op.get("totalCount").toString());
final String opType = op.get("opType").toString();
final int expected;
if (opType.equals(TopConf.ALL_CMDS)) {
- expected = 2*NUM_OPS;
+ expected = 2 * NUM_OPS + 2;
+ } else if (opType.equals("datanodeReport")) {
+ expected = 2;
} else {
expected = NUM_OPS;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-14651. Update okhttp
version to 2.7.5. Contributed by Ray Chiang and John Zhuge.
Posted by kk...@apache.org.
HADOOP-14651. Update okhttp version to 2.7.5. Contributed by Ray Chiang and John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aca46e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aca46e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aca46e5
Branch: refs/heads/YARN-6592
Commit: 8aca46e53f5962217851dc2103211f99a4ef7258
Parents: f40dbc1
Author: John Zhuge <jz...@cloudera.com>
Authored: Wed Sep 13 20:36:31 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Fri Sep 29 12:15:06 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 3 ++-
hadoop-tools/hadoop-azure-datalake/pom.xml | 7 ++++---
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aca46e5/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 915bd4e..66d2eb4 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -102,6 +102,7 @@
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
<mssql.version>6.2.1.jre7</mssql.version>
+ <okhttp.version>2.7.5</okhttp.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
@@ -147,7 +148,7 @@
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>okhttp</artifactId>
- <version>2.4.0</version>
+ <version>${okhttp.version}</version>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aca46e5/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index e32f9fc..660bfb9 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -29,7 +29,7 @@
</description>
<packaging>jar</packaging>
<properties>
- <okHttpVersion>2.4.0</okHttpVersion>
+ <okHttpVersion>${okhttp.version}</okHttpVersion>
<minimalJsonVersion>0.9.1</minimalJsonVersion>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
@@ -120,7 +120,8 @@
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>okhttp</artifactId>
- <version>2.4.0</version>
+ <version>${okhttp.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -142,7 +143,7 @@
<dependency>
<groupId>com.squareup.okhttp</groupId>
<artifactId>mockwebserver</artifactId>
- <version>2.4.0</version>
+ <version>${okhttp.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HADOOP-14768. Honoring sticky bit
during Deletion when authorization is enabled in WASB Contributed by Varada
Hemeswari
Posted by kk...@apache.org.
HADOOP-14768. Honoring sticky bit during Deletion when authorization is enabled in WASB
Contributed by Varada Hemeswari
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a530e7ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a530e7ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a530e7ab
Branch: refs/heads/YARN-6592
Commit: a530e7ab3b3f5bd71143a91266b46787962ac532
Parents: 8e1bd11
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 28 19:52:56 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 28 19:52:56 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 522 +++++++++++++++--
...stNativeAzureFSAuthWithBlobSpecificKeys.java | 2 +-
.../ITestNativeAzureFSAuthorizationCaching.java | 2 +-
...veAzureFileSystemAuthorizationWithOwner.java | 122 ----
.../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 284 +++++-----
.../TestNativeAzureFileSystemAuthorization.java | 555 +++++++++++++++++--
6 files changed, 1136 insertions(+), 351 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 280c0e0..5f86f84 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -40,6 +40,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Arrays;
import java.util.List;
+import java.util.Stack;
+import java.util.HashMap;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
@@ -80,6 +82,8 @@ import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.azure.NativeAzureFileSystemHelper.*;
+
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.microsoft.azure.storage.StorageException;
@@ -1849,31 +1853,256 @@ public class NativeAzureFileSystem extends FileSystem {
}
/**
- * Delete the specified file or folder. The parameter
- * skipParentFolderLastModifiedTimeUpdate
- * is used in the case of atomic folder rename redo. In that case, there is
- * a lease on the parent folder, so (without reworking the code) modifying
- * the parent folder update time will fail because of a conflict with the
- * lease. Since we are going to delete the folder soon anyway so accurate
- * modified time is not necessary, it's easier to just skip
- * the modified time update.
- *
- * @param f file path to be deleted.
- * @param recursive specify deleting recursively or not.
- * @param skipParentFolderLastModifiedTimeUpdate If true, don't update the folder last
- * modified time.
- * @return true if and only if the file is deleted
- * @throws IOException Thrown when fail to delete file or directory.
+ * Delete file or folder with authorization checks. Most of the code
+ * is duplicate of the actual delete implementation and will be merged
+ * once the performance and funcional aspects are guaranteed not to
+ * regress existing delete semantics.
*/
- public boolean delete(Path f, boolean recursive,
+ private boolean deleteWithAuthEnabled(Path f, boolean recursive,
boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
- LOG.debug("Deleting file: {}", f.toString());
+ LOG.debug("Deleting file: {}", f);
Path absolutePath = makeAbsolute(f);
Path parentPath = absolutePath.getParent();
- performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
+ // If delete is issued for 'root', parentPath will be null
+ // In that case, we perform auth check for root itself before
+ // proceeding for deleting contents under root.
+ if (parentPath != null) {
+ performAuthCheck(parentPath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
+ } else {
+ performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
+ }
+
+ String key = pathToKey(absolutePath);
+
+ // Capture the metadata for the path.
+ FileMetadata metaFile = null;
+ try {
+ metaFile = store.retrieveMetadata(key);
+ } catch (IOException e) {
+
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+
+ return false;
+ }
+ throw e;
+ }
+
+ if (null == metaFile) {
+ // The path to be deleted does not exist.
+ return false;
+ }
+
+ FileMetadata parentMetadata = null;
+ String parentKey = null;
+ if (parentPath != null) {
+ parentKey = pathToKey(parentPath);
+
+ try {
+ parentMetadata = store.retrieveMetadata(parentKey);
+ } catch (IOException e) {
+ Throwable innerException = checkForAzureStorageException(e);
+ if (innerException instanceof StorageException) {
+ // Invalid State.
+ // A FileNotFoundException is not thrown here as the API returns false
+ // if the file not present. But not retrieving metadata here is an
+ // unrecoverable state and can only happen if there is a race condition
+ // hence throwing a IOException
+ if (isFileNotFoundException((StorageException) innerException)) {
+ throw new IOException("File " + f + " has a parent directory "
+ + parentPath + " whose metadata cannot be retrieved. Can't resolve");
+ }
+ }
+ throw e;
+ }
+
+ // Same case as unable to retrieve metadata
+ if (parentMetadata == null) {
+ throw new IOException("File " + f + " has a parent directory "
+ + parentPath + " whose metadata cannot be retrieved. Can't resolve");
+ }
+
+ if (!parentMetadata.isDir()) {
+ // Invalid state: the parent path is actually a file. Throw.
+ throw new AzureException("File " + f + " has a parent directory "
+ + parentPath + " which is also a file. Can't resolve.");
+ }
+ }
+
+ // The path exists, determine if it is a folder containing objects,
+ // an empty folder, or a simple file and take the appropriate actions.
+ if (!metaFile.isDir()) {
+ // The path specifies a file. We need to check the parent path
+ // to make sure it's a proper materialized directory before we
+ // delete the file. Otherwise we may get into a situation where
+ // the file we were deleting was the last one in an implicit directory
+ // (e.g. the blob store only contains the blob a/b and there's no
+ // corresponding directory blob a) and that would implicitly delete
+ // the directory as well, which is not correct.
+
+ if (parentPath != null && parentPath.getParent() != null) {// Not root
+
+ if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
+ LOG.debug("Found an implicit parent directory while trying to"
+ + " delete the file {}. Creating the directory blob for"
+ + " it in {}.", f, parentKey);
+
+ store.storeEmptyFolder(parentKey,
+ createPermissionStatus(FsPermission.getDefault()));
+ } else {
+ if (!skipParentFolderLastModifiedTimeUpdate) {
+ updateParentFolderLastModifiedTime(key);
+ }
+ }
+ }
+
+ // check if the file can be deleted based on sticky bit check
+ // This check will be performed only when authorization is enabled
+ if (isStickyBitCheckViolated(metaFile, parentMetadata)) {
+ throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
+ + "File %s cannot be deleted.", parentPath, f));
+ }
+
+ try {
+ if (store.delete(key)) {
+ instrumentation.fileDeleted();
+ } else {
+ return false;
+ }
+ } catch(IOException e) {
+
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+ return false;
+ }
+
+ throw e;
+ }
+ } else {
+ // The path specifies a folder. Recursively delete all entries under the
+ // folder.
+ LOG.debug("Directory Delete encountered: {}", f);
+ if (parentPath != null && parentPath.getParent() != null) {
+
+ if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) {
+ LOG.debug("Found an implicit parent directory while trying to"
+ + " delete the directory {}. Creating the directory blob for"
+ + " it in {}. ", f, parentKey);
+
+ store.storeEmptyFolder(parentKey,
+ createPermissionStatus(FsPermission.getDefault()));
+ }
+ }
+
+ // check if the folder can be deleted based on sticky bit check on parent
+ // This check will be performed only when authorization is enabled.
+ if (!metaFile.getKey().equals("/")
+ && isStickyBitCheckViolated(metaFile, parentMetadata)) {
+
+ throw new WasbAuthorizationException(String.format("%s has sticky bit set. "
+ + "File %s cannot be deleted.", parentPath, f));
+ }
+
+ // Iterate through folder contents and get the list of files
+ // and folders that can be deleted. We might encounter IOException
+ // while listing blobs. In such cases, we return false.
+ ArrayList<FileMetadata> fileMetadataList = new ArrayList<>();
+ boolean isPartialDelete = false;
+
+ // Start time for list operation
+ long start = Time.monotonicNow();
+
+ try {
+ // Get list of files/folders that can be deleted
+ // based on authorization checks and stickybit checks
+ isPartialDelete = getFolderContentsToDelete(metaFile, fileMetadataList);
+ } catch (IOException e) {
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+ return false;
+ }
+ throw e;
+ }
+
+ long end = Time.monotonicNow();
+ LOG.debug("Time taken to list {} blobs for delete operation: {} ms",
+ fileMetadataList.size(), (end - start));
+
+ // Here contents holds the list of metadata of the files and folders that can be deleted
+ // under the path that is requested for delete(excluding itself).
+ final FileMetadata[] contents = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]);
+
+ if (contents.length > 0 && !recursive) {
+ // The folder is non-empty and recursive delete was not specified.
+ // Throw an exception indicating that a non-recursive delete was
+ // specified for a non-empty folder.
+ throw new IOException("Non-recursive delete of non-empty directory "
+ + f);
+ }
+
+ // Delete all files / folders in current directory stored as list in 'contents'.
+ AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
+ @Override
+ public boolean execute(FileMetadata file) throws IOException{
+ if (!deleteFile(file.getKey(), file.isDir())) {
+ LOG.warn("Attempt to delete non-existent {} {}",
+ file.isDir() ? "directory" : "file",
+ file.getKey());
+ }
+ return true;
+ }
+ };
+
+ AzureFileSystemThreadPoolExecutor executor = getThreadPoolExecutor(this.deleteThreadCount,
+ "AzureBlobDeleteThread", "Delete", key, AZURE_DELETE_THREADS);
+
+ if (!executor.executeParallel(contents, task)) {
+ LOG.error("Failed to delete files / subfolders in blob {}", key);
+ return false;
+ }
+
+ if (metaFile.getKey().equals("/")) {
+ LOG.error("Cannot delete root directory {}", f);
+ return false;
+ }
+
+ // Delete the current directory if all underlying contents are deleted
+ if (isPartialDelete || (store.retrieveMetadata(metaFile.getKey()) != null
+ && !deleteFile(metaFile.getKey(), metaFile.isDir()))) {
+ LOG.error("Failed delete directory : {}", f);
+ return false;
+ }
+
+ // Update parent directory last modified time
+ Path parent = absolutePath.getParent();
+ if (parent != null && parent.getParent() != null) { // not root
+ if (!skipParentFolderLastModifiedTimeUpdate) {
+ updateParentFolderLastModifiedTime(key);
+ }
+ }
+ }
+
+ // File or directory was successfully deleted.
+ LOG.debug("Delete Successful for : {}", f);
+ return true;
+ }
+
+ private boolean deleteWithoutAuth(Path f, boolean recursive,
+ boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
+
+ LOG.debug("Deleting file: {}", f);
+
+ Path absolutePath = makeAbsolute(f);
+ Path parentPath = absolutePath.getParent();
String key = pathToKey(absolutePath);
@@ -1884,10 +2113,10 @@ public class NativeAzureFileSystem extends FileSystem {
metaFile = store.retrieveMetadata(key);
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ && isFileNotFoundException((StorageException) innerException)) {
return false;
}
@@ -1918,7 +2147,7 @@ public class NativeAzureFileSystem extends FileSystem {
parentMetadata = store.retrieveMetadata(parentKey);
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException) {
// Invalid State.
@@ -1926,7 +2155,7 @@ public class NativeAzureFileSystem extends FileSystem {
// if the file not present. But not retrieving metadata here is an
// unrecoverable state and can only happen if there is a race condition
// hence throwing a IOException
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ if (isFileNotFoundException((StorageException) innerException)) {
throw new IOException("File " + f + " has a parent directory "
+ parentPath + " whose metadata cannot be retrieved. Can't resolve");
}
@@ -1972,10 +2201,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
} catch(IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ && isFileNotFoundException((StorageException) innerException)) {
return false;
}
@@ -1984,7 +2213,7 @@ public class NativeAzureFileSystem extends FileSystem {
} else {
// The path specifies a folder. Recursively delete all entries under the
// folder.
- LOG.debug("Directory Delete encountered: {}", f.toString());
+ LOG.debug("Directory Delete encountered: {}", f);
if (parentPath.getParent() != null) {
String parentKey = pathToKey(parentPath);
FileMetadata parentMetadata = null;
@@ -1993,7 +2222,7 @@ public class NativeAzureFileSystem extends FileSystem {
parentMetadata = store.retrieveMetadata(parentKey);
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException) {
// Invalid State.
@@ -2001,7 +2230,7 @@ public class NativeAzureFileSystem extends FileSystem {
// if the file not present. But not retrieving metadata here is an
// unrecoverable state and can only happen if there is a race condition
// hence throwing a IOException
- if (NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ if (isFileNotFoundException((StorageException) innerException)) {
throw new IOException("File " + f + " has a parent directory "
+ parentPath + " whose metadata cannot be retrieved. Can't resolve");
}
@@ -2046,10 +2275,10 @@ public class NativeAzureFileSystem extends FileSystem {
}
priorLastKey = listing.getPriorLastKey();
} catch (IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
+ Throwable innerException = checkForAzureStorageException(e);
if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
+ && isFileNotFoundException((StorageException) innerException)) {
return false;
}
@@ -2067,22 +2296,7 @@ public class NativeAzureFileSystem extends FileSystem {
// The folder is non-empty and recursive delete was not specified.
// Throw an exception indicating that a non-recursive delete was
// specified for a non-empty folder.
- throw new IOException("Non-recursive delete of non-empty directory "
- + f.toString());
- }
- else {
- // Check write-permissions on sub-tree including current folder
- // NOTE: Ideally the subtree needs read-write-execute access check.
- // But we will simplify it to write-access check.
- if (metaFile.isDir()) { // the absolute-path
- performAuthCheck(absolutePath, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- }
- for (FileMetadata meta : contents) {
- if (meta.isDir()) {
- Path subTreeDir = keyToPath(meta.getKey());
- performAuthCheck(subTreeDir, WasbAuthorizationOperations.WRITE, "delete", absolutePath);
- }
- }
+ throw new IOException("Non-recursive delete of non-empty directory "+ f);
}
}
@@ -2108,8 +2322,9 @@ public class NativeAzureFileSystem extends FileSystem {
}
// Delete the current directory
- if (store.retrieveMetadata(metaFile.getKey()) != null && !deleteFile(metaFile.getKey(), metaFile.isDir())) {
- LOG.error("Failed delete directory {}", f.toString());
+ if (store.retrieveMetadata(metaFile.getKey()) != null
+ && !deleteFile(metaFile.getKey(), metaFile.isDir())) {
+ LOG.error("Failed delete directory : {}", f);
return false;
}
@@ -2123,16 +2338,227 @@ public class NativeAzureFileSystem extends FileSystem {
}
// File or directory was successfully deleted.
- LOG.debug("Delete Successful for : {}", f.toString());
+ LOG.debug("Delete Successful for : {}", f);
return true;
}
+ /**
+ * Delete the specified file or folder. The parameter
+ * skipParentFolderLastModifiedTimeUpdate
+ * is used in the case of atomic folder rename redo. In that case, there is
+ * a lease on the parent folder, so (without reworking the code) modifying
+ * the parent folder update time will fail because of a conflict with the
+ * lease. Since we are going to delete the folder soon anyway so accurate
+ * modified time is not necessary, it's easier to just skip
+ * the modified time update.
+ *
+ * @param f file path to be deleted.
+ * @param recursive specify deleting recursively or not.
+ * @param skipParentFolderLastModifiedTimeUpdate If true, don't update the folder last
+ * modified time.
+ * @return true if and only if the file is deleted
+ * @throws IOException Thrown when fail to delete file or directory.
+ */
+ public boolean delete(Path f, boolean recursive,
+ boolean skipParentFolderLastModifiedTimeUpdate) throws IOException {
+
+ if (this.azureAuthorization) {
+ return deleteWithAuthEnabled(f, recursive,
+ skipParentFolderLastModifiedTimeUpdate);
+ } else {
+ return deleteWithoutAuth(f, recursive,
+ skipParentFolderLastModifiedTimeUpdate);
+ }
+ }
+
public AzureFileSystemThreadPoolExecutor getThreadPoolExecutor(int threadCount,
String threadNamePrefix, String operation, String key, String config) {
return new AzureFileSystemThreadPoolExecutor(threadCount, threadNamePrefix, operation, key, config);
}
/**
+ * Gets list of contents that can be deleted based on authorization check calls
+ * performed on the sub-tree for the folderToDelete.
+ *
+ * @param folderToDelete - metadata of the folder whose delete is requested.
+ * @param finalList - list of metadata of all files/folders that can be deleted .
+ *
+ * @return 'true' only if all the contents of the folderToDelete can be deleted
+ * @throws IOException Thrown when current user cannot be retrieved.
+ */
+ private boolean getFolderContentsToDelete(FileMetadata folderToDelete,
+ ArrayList<FileMetadata> finalList) throws IOException {
+
+ final int maxListingDepth = 1;
+ Stack<FileMetadata> foldersToProcess = new Stack<FileMetadata>();
+ HashMap<String, FileMetadata> folderContentsMap = new HashMap<String, FileMetadata>();
+
+ boolean isPartialDelete = false;
+
+ Path pathToDelete = makeAbsolute(keyToPath(folderToDelete.getKey()));
+ foldersToProcess.push(folderToDelete);
+
+ while (!foldersToProcess.empty()) {
+
+ FileMetadata currentFolder = foldersToProcess.pop();
+ Path currentPath = makeAbsolute(keyToPath(currentFolder.getKey()));
+ boolean canDeleteChildren = true;
+
+ // If authorization is enabled, check for 'write' permission on current folder
+ // This check maps to subfolders 'write' check for deleting contents recursively.
+ try {
+ performAuthCheck(currentPath, WasbAuthorizationOperations.WRITE, "delete", pathToDelete);
+ } catch (WasbAuthorizationException we) {
+ LOG.debug("Authorization check failed for {}", currentPath);
+ // We cannot delete the children of currentFolder since 'write' check on parent failed
+ canDeleteChildren = false;
+ }
+
+ if (canDeleteChildren) {
+
+ // get immediate children list
+ ArrayList<FileMetadata> fileMetadataList = getChildrenMetadata(currentFolder.getKey(),
+ maxListingDepth);
+
+ // Process children of currentFolder and add them to list of contents
+ // that can be deleted. We Perform stickybit check on every file and
+ // folder under currentFolder in case stickybit is set on currentFolder.
+ for (FileMetadata childItem : fileMetadataList) {
+ if (isStickyBitCheckViolated(childItem, currentFolder, false)) {
+ // Stickybit check failed for the childItem that is being processed.
+ // This file/folder cannot be deleted and neither can the parent paths be deleted.
+ // Remove parent paths from list of contents that can be deleted.
+ canDeleteChildren = false;
+ Path filePath = makeAbsolute(keyToPath(childItem.getKey()));
+ LOG.error("User does not have permissions to delete {}. "
+ + "Parent directory has sticky bit set.", filePath);
+ } else {
+ // push the child directories to the stack to process their contents
+ if (childItem.isDir()) {
+ foldersToProcess.push(childItem);
+ }
+ // Add items to list of contents that can be deleted.
+ folderContentsMap.put(childItem.getKey(), childItem);
+ }
+ }
+
+ } else {
+ // Cannot delete children since parent permission check has not passed and
+ // if there are files/folders under currentFolder they will not be deleted.
+ LOG.error("Authorization check failed. Files or folders under {} "
+ + "will not be processed for deletion.", currentPath);
+ }
+
+ if (!canDeleteChildren) {
+ // We reach here if
+ // 1. cannot delete children since 'write' check on parent failed or
+ // 2. One of the files under the current folder cannot be deleted due to stickybit check.
+ // In this case we remove all the parent paths from the list of contents
+ // that can be deleted till we reach the original path of delete request
+ String pathToRemove = currentFolder.getKey();
+ while (!pathToRemove.equals(folderToDelete.getKey())) {
+ if (folderContentsMap.containsKey(pathToRemove)) {
+ LOG.debug("Cannot delete {} since some of its contents "
+ + "cannot be deleted", pathToRemove);
+ folderContentsMap.remove(pathToRemove);
+ }
+ Path parentPath = keyToPath(pathToRemove).getParent();
+ pathToRemove = pathToKey(parentPath);
+ }
+ // Since one or more files/folders cannot be deleted return value should indicate
+ // partial delete, so that the delete on the path requested by user is not performed
+ isPartialDelete = true;
+ }
+ }
+
+ // final list of contents that can be deleted
+ for (HashMap.Entry<String, FileMetadata> entry : folderContentsMap.entrySet()) {
+ finalList.add(entry.getValue());
+ }
+
+ return isPartialDelete;
+ }
+
+ private ArrayList<FileMetadata> getChildrenMetadata(String key, int maxListingDepth)
+ throws IOException {
+
+ String priorLastKey = null;
+ ArrayList<FileMetadata> fileMetadataList = new ArrayList<FileMetadata>();
+ do {
+ PartialListing listing = store.listAll(key, AZURE_LIST_ALL,
+ maxListingDepth, priorLastKey);
+ for (FileMetadata file : listing.getFiles()) {
+ fileMetadataList.add(file);
+ }
+ priorLastKey = listing.getPriorLastKey();
+ } while (priorLastKey != null);
+
+ return fileMetadataList;
+ }
+
+ private boolean isStickyBitCheckViolated(FileMetadata metaData,
+ FileMetadata parentMetadata, boolean throwOnException) throws IOException {
+ try {
+ return isStickyBitCheckViolated(metaData, parentMetadata);
+ } catch (FileNotFoundException ex) {
+ if (throwOnException) {
+ throw ex;
+ } else {
+ LOG.debug("Encountered FileNotFoundException while performing "
+ + "stickybit check operation for {}", metaData.getKey());
+ // swallow exception and return that stickyBit check has been violated
+ return true;
+ }
+ }
+ }
+
+ /**
+ * Checks if the Current user is not permitted access to a file/folder when
+ * sticky bit is set on parent path. Only the owner of parent path
+ * and owner of the file/folder itself are permitted to perform certain
+ * operations on file/folder based on sticky bit check. Sticky bit check will
+ * be performed only when authorization is enabled.
+ *
+ * @param metaData - metadata of the file/folder whose parent has sticky bit set.
+ * @param parentMetadata - metadata of the parent.
+ *
+ * @return true if Current user violates stickybit check
+ * @throws IOException Thrown when current user cannot be retrieved.
+ */
+ private boolean isStickyBitCheckViolated(FileMetadata metaData,
+ FileMetadata parentMetadata) throws IOException {
+
+ // In case stickybit check should not be performed,
+ // return value should indicate stickybit check is not violated.
+ if (!this.azureAuthorization) {
+ return false;
+ }
+
+ // This should never happen when the sticky bit check is invoked.
+ if (parentMetadata == null) {
+ throw new FileNotFoundException(
+ String.format("Parent metadata for '%s' not found!", metaData.getKey()));
+ }
+
+ // stickybit is not set on parent and hence cannot be violated
+ if (!parentMetadata.getPermissionStatus().getPermission().getStickyBit()) {
+ return false;
+ }
+
+ String currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
+ String parentDirectoryOwner = parentMetadata.getPermissionStatus().getUserName();
+ String currentFileOwner = metaData.getPermissionStatus().getUserName();
+
+ // Files/Folders with no owner set will not pass stickybit check
+ if ((parentDirectoryOwner.equalsIgnoreCase(currentUser))
+ || currentFileOwner.equalsIgnoreCase(currentUser)) {
+
+ return false;
+ }
+ return true;
+ }
+
+ /**
* Delete the specified file or directory and increment metrics.
* If the file or directory does not exist, the operation returns false.
* @param path the path to a file or directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
index d7e4831..0f3d127 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthWithBlobSpecificKeys.java
@@ -27,7 +27,7 @@ import static org.apache.hadoop.fs.azure.SecureStorageInterfaceImpl.KEY_USE_CONT
* to access storage.
*/
public class ITestNativeAzureFSAuthWithBlobSpecificKeys
- extends ITestNativeAzureFileSystemAuthorizationWithOwner {
+ extends TestNativeAzureFileSystemAuthorization {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
index c73b1cc..138063d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFSAuthorizationCaching.java
@@ -27,7 +27,7 @@ import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACH
* Test class to hold all WASB authorization caching related tests.
*/
public class ITestNativeAzureFSAuthorizationCaching
- extends ITestNativeAzureFileSystemAuthorizationWithOwner {
+ extends TestNativeAzureFileSystemAuthorization {
private static final int DUMMY_TTL_VALUE = 5000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java
deleted file mode 100644
index 3ec42f0..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAuthorizationWithOwner.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import java.security.PrivilegedExceptionAction;
-
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test class that runs wasb authorization tests with owner check enabled.
- */
-public class ITestNativeAzureFileSystemAuthorizationWithOwner
- extends TestNativeAzureFileSystemAuthorization {
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- authorizer.init(fs.getConf(), true);
- }
-
- /**
- * Test case when owner matches current user.
- */
- @Test
- public void testOwnerPermissionPositive() throws Throwable {
-
- Path parentDir = new Path("/testOwnerPermissionPositive");
- Path testPath = new Path(parentDir, "test.data");
-
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
- // additional rule used for assertPathExists
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.READ.toString(), true);
- fs.updateWasbAuthorizer(authorizer);
-
- try {
- // creates parentDir with owner as current user
- fs.mkdirs(parentDir);
- ContractTestUtils.assertPathExists(fs, "parentDir does not exist", parentDir);
-
- fs.create(testPath);
- fs.getFileStatus(testPath);
- ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
-
- } finally {
- allowRecursiveDelete(fs, parentDir.toString());
- fs.delete(parentDir, true);
- }
- }
-
- /**
- * Negative test case for owner does not match current user.
- */
- @Test
- public void testOwnerPermissionNegative() throws Throwable {
- expectedEx.expect(WasbAuthorizationException.class);
-
- Path parentDir = new Path("/testOwnerPermissionNegative");
- Path childDir = new Path(parentDir, "childDir");
-
- setExpectedFailureMessage("mkdirs", childDir);
-
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
-
- fs.updateWasbAuthorizer(authorizer);
-
- try{
- fs.mkdirs(parentDir);
- UserGroupInformation ugiSuperUser = UserGroupInformation.createUserForTesting(
- "testuser", new String[] {});
-
- ugiSuperUser.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- fs.mkdirs(childDir);
- return null;
- }
- });
-
- } finally {
- allowRecursiveDelete(fs, parentDir.toString());
- fs.delete(parentDir, true);
- }
- }
-
- /**
- * Test to verify that retrieving owner information does not
- * throw when file/folder does not exist.
- */
- @Test
- public void testRetrievingOwnerDoesNotFailWhenFileDoesNotExist() throws Throwable {
-
- Path testdirectory = new Path("/testDirectory123454565");
-
- String owner = fs.getOwnerForPath(testdirectory);
- assertEquals("", owner);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
index 7354499..1f5072d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java
@@ -34,165 +34,183 @@ import org.apache.hadoop.fs.Path;
public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface {
- private Map<AuthorizationComponent, Boolean> authRules;
- private boolean performOwnerMatch;
- private CachingAuthorizer<CachedAuthorizerEntry, Boolean> cache;
-
- // The full qualified URL to the root directory
- private String qualifiedPrefixUrl;
-
- public MockWasbAuthorizerImpl(NativeAzureFileSystem fs) {
- qualifiedPrefixUrl = new Path("/").makeQualified(fs.getUri(),
- fs.getWorkingDirectory())
- .toString().replaceAll("/$", "");
- cache = new CachingAuthorizer<>(TimeUnit.MINUTES.convert(5L, TimeUnit.MINUTES), "AUTHORIZATION");
- }
-
- @Override
- public void init(Configuration conf) {
- init(conf, false);
- }
-
- /*
- authorization matches owner with currentUserShortName while evaluating auth rules
- if currentUserShortName is set to a string that is not empty
- */
- public void init(Configuration conf, boolean matchOwner) {
- cache.init(conf);
- authRules = new HashMap<>();
- this.performOwnerMatch = matchOwner;
- }
-
- public void addAuthRule(String wasbAbsolutePath,
- String accessType, boolean access) {
- wasbAbsolutePath = qualifiedPrefixUrl + wasbAbsolutePath;
- AuthorizationComponent component = wasbAbsolutePath.endsWith("*")
- ? new AuthorizationComponent("^" + wasbAbsolutePath.replace("*", ".*"),
- accessType)
- : new AuthorizationComponent(wasbAbsolutePath, accessType);
-
- this.authRules.put(component, access);
- }
-
- @Override
- public boolean authorize(String wasbAbsolutePath,
- String accessType,
- String owner)
- throws WasbAuthorizationException {
-
- if (wasbAbsolutePath.endsWith(
- NativeAzureFileSystem.FolderRenamePending.SUFFIX)) {
- return true;
- }
+ private Map<AuthorizationComponent, Boolean> authRules;
+ private CachingAuthorizer<CachedAuthorizerEntry, Boolean> cache;
+
+ // The full qualified URL to the root directory
+ private String qualifiedPrefixUrl;
- CachedAuthorizerEntry cacheKey = new CachedAuthorizerEntry(wasbAbsolutePath, accessType, owner);
- Boolean cacheresult = cache.get(cacheKey);
- if (cacheresult != null) {
- return cacheresult;
+ public MockWasbAuthorizerImpl(NativeAzureFileSystem fs) {
+ qualifiedPrefixUrl = new Path("/").makeQualified(fs.getUri(),
+ fs.getWorkingDirectory())
+ .toString().replaceAll("/$", "");
+ cache = new CachingAuthorizer<>(TimeUnit.MINUTES.convert(5L, TimeUnit.MINUTES), "AUTHORIZATION");
}
- boolean authorizeresult = authorizeInternal(wasbAbsolutePath, accessType, owner);
- cache.put(cacheKey, authorizeresult);
+ @Override
+ public void init(Configuration conf) {
+ cache.init(conf);
+ authRules = new HashMap<>();
+ }
- return authorizeresult;
- }
+ public void addAuthRuleForOwner(String wasbAbsolutePath,
+ String accessType, boolean access) {
+ addAuthRule(wasbAbsolutePath, accessType, "owner", access);
+ }
- private boolean authorizeInternal(String wasbAbsolutePath, String accessType, String owner)
- throws WasbAuthorizationException {
+ public void addAuthRule(String wasbAbsolutePath,
+ String accessType, String user, boolean access) {
+ wasbAbsolutePath = qualifiedPrefixUrl + wasbAbsolutePath;
+ AuthorizationComponent component = wasbAbsolutePath.endsWith("*")
+ ? new AuthorizationComponent("^" + wasbAbsolutePath.replace("*", ".*"),
+ accessType, user)
+ : new AuthorizationComponent(wasbAbsolutePath, accessType, user);
- String currentUserShortName = "";
- if (this.performOwnerMatch) {
- try {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- currentUserShortName = ugi.getShortUserName();
- } catch (Exception e) {
- //no op
- }
+ this.authRules.put(component, access);
}
- // In case of root("/"), owner match does not happen because owner is returned as empty string.
- // we try to force owner match just for purpose of tests to make sure all operations work seemlessly with owner.
- if (this.performOwnerMatch
- && StringUtils.equalsIgnoreCase(wasbAbsolutePath,
- qualifiedPrefixUrl + "/")) {
- owner = currentUserShortName;
- }
+ @Override
+ public boolean authorize(String wasbAbsolutePath,
+ String accessType,
+ String owner)
+ throws WasbAuthorizationException {
+
+ if (wasbAbsolutePath.endsWith(
+ NativeAzureFileSystem.FolderRenamePending.SUFFIX)) {
+ return true;
+ }
- boolean shouldEvaluateOwnerAccess = owner != null && !owner.isEmpty()
- && this.performOwnerMatch;
-
- boolean isOwnerMatch = StringUtils.equalsIgnoreCase(currentUserShortName,
- owner);
-
- AuthorizationComponent component =
- new AuthorizationComponent(wasbAbsolutePath, accessType);
-
- if (authRules.containsKey(component)) {
- return shouldEvaluateOwnerAccess ? isOwnerMatch && authRules.get(
- component) : authRules.get(component);
- } else {
- // Regex-pattern match if we don't have a straight match
- for (Map.Entry<AuthorizationComponent, Boolean> entry : authRules.entrySet()) {
- AuthorizationComponent key = entry.getKey();
- String keyPath = key.getWasbAbsolutePath();
- String keyAccess = key.getAccessType();
-
- if (keyPath.endsWith("*") && Pattern.matches(keyPath, wasbAbsolutePath)
- && keyAccess.equals(accessType)) {
- return shouldEvaluateOwnerAccess
- ? isOwnerMatch && entry.getValue()
- : entry.getValue();
+ CachedAuthorizerEntry cacheKey = new CachedAuthorizerEntry(wasbAbsolutePath, accessType, owner);
+ Boolean cacheresult = cache.get(cacheKey);
+ if (cacheresult != null) {
+ return cacheresult;
}
- }
- return false;
+
+ boolean authorizeresult = authorizeInternal(wasbAbsolutePath, accessType, owner);
+ cache.put(cacheKey, authorizeresult);
+
+ return authorizeresult;
}
- }
- public void deleteAllAuthRules() {
- authRules.clear();
- cache.clear();
- }
+ private boolean authorizeInternal(String wasbAbsolutePath, String accessType, String owner)
+ throws WasbAuthorizationException {
- private static class AuthorizationComponent {
+ String currentUserShortName = "";
+ try {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ currentUserShortName = ugi.getShortUserName();
+ } catch (Exception e) {
+ //no op
+ }
- private final String wasbAbsolutePath;
- private final String accessType;
+ // In case of root("/"), owner match does not happen
+ // because owner is returned as empty string.
+ // we try to force owner match just for purpose of tests
+ // to make sure all operations work seemlessly with owner.
+ if (StringUtils.equalsIgnoreCase(wasbAbsolutePath, qualifiedPrefixUrl + "/")) {
+ owner = currentUserShortName;
+ }
- AuthorizationComponent(String wasbAbsolutePath,
- String accessType) {
- this.wasbAbsolutePath = wasbAbsolutePath;
- this.accessType = accessType;
+ AuthorizationComponent component = new AuthorizationComponent(wasbAbsolutePath,
+ accessType, currentUserShortName);
+
+ return processRules(authRules, component, owner);
}
- @Override
- public int hashCode() {
- return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode();
+ private boolean processRules(Map<AuthorizationComponent, Boolean> authRules,
+ AuthorizationComponent component, String owner) {
+
+ // Direct match of rules and access request
+ if (authRules.containsKey(component)) {
+ return authRules.get(component);
+ } else {
+ // Regex-pattern match if we don't have a straight match for path
+ // Current user match if we don't have a owner match
+ for (Map.Entry<AuthorizationComponent, Boolean> entry : authRules.entrySet()) {
+ AuthorizationComponent key = entry.getKey();
+ String keyPath = key.getWasbAbsolutePath();
+ String keyAccess = key.getAccessType();
+ String keyUser = key.getUser();
+
+ boolean foundMatchingOwnerRule = keyPath.equals(component.getWasbAbsolutePath())
+ && keyAccess.equals(component.getAccessType())
+ && keyUser.equalsIgnoreCase("owner")
+ && owner.equals(component.getUser());
+
+ boolean foundMatchingPatternRule = keyPath.endsWith("*")
+ && Pattern.matches(keyPath, component.getWasbAbsolutePath())
+ && keyAccess.equals(component.getAccessType())
+ && keyUser.equalsIgnoreCase(component.getUser());
+
+ boolean foundMatchingPatternOwnerRule = keyPath.endsWith("*")
+ && Pattern.matches(keyPath, component.getWasbAbsolutePath())
+ && keyAccess.equals(component.getAccessType())
+ && keyUser.equalsIgnoreCase("owner")
+ && owner.equals(component.getUser());
+
+ if (foundMatchingOwnerRule
+ || foundMatchingPatternRule
+ || foundMatchingPatternOwnerRule) {
+ return entry.getValue();
+ }
+ }
+ return false;
+ }
}
- @Override
- public boolean equals(Object obj) {
+ public void deleteAllAuthRules() {
+ authRules.clear();
+ cache.clear();
+ }
+
+ private static class AuthorizationComponent {
+
+ private final String wasbAbsolutePath;
+ private final String accessType;
+ private final String user;
- if (obj == this) {
- return true;
+ AuthorizationComponent(String wasbAbsolutePath,
+ String accessType, String user) {
+ this.wasbAbsolutePath = wasbAbsolutePath;
+ this.accessType = accessType;
+ this.user = user;
}
- if (obj == null
- || !(obj instanceof AuthorizationComponent)) {
- return false;
+ @Override
+ public int hashCode() {
+ return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode();
}
- return ((AuthorizationComponent) obj).
+ @Override
+ public boolean equals(Object obj) {
+
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj == null
+ || !(obj instanceof AuthorizationComponent)) {
+ return false;
+ }
+
+ return ((AuthorizationComponent) obj).
getWasbAbsolutePath().equals(this.wasbAbsolutePath)
&& ((AuthorizationComponent) obj).
- getAccessType().equals(this.accessType);
- }
+ getAccessType().equals(this.accessType)
+ && ((AuthorizationComponent) obj).
+ getUser().equals(this.user);
+ }
- public String getWasbAbsolutePath() {
- return this.wasbAbsolutePath;
- }
+ public String getWasbAbsolutePath() {
+ return this.wasbAbsolutePath;
+ }
- public String getAccessType() {
- return accessType;
- }
- }
+ public String getAccessType() {
+ return accessType;
+ }
+
+ public String getUser() {
+ return user;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a530e7ab/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
index 4bf6f04..2129b33 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.fs.azure;
import java.security.PrivilegedExceptionAction;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -47,6 +49,13 @@ public class TestNativeAzureFileSystemAuthorization
@VisibleForTesting
protected MockWasbAuthorizerImpl authorizer;
+ @VisibleForTesting
+ protected static final short STICKYBIT_PERMISSION_CONSTANT = 1700;
+ @VisibleForTesting
+ protected static final String READ = WasbAuthorizationOperations.READ.toString();
+ @VisibleForTesting
+ protected static final String WRITE = WasbAuthorizationOperations.WRITE.toString();
+
@Override
public Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
@@ -80,14 +89,16 @@ public class TestNativeAzureFileSystemAuthorization
/**
* Setup up permissions to allow a recursive delete for cleanup purposes.
*/
- protected void allowRecursiveDelete(NativeAzureFileSystem fs, String path) {
+ protected void allowRecursiveDelete(NativeAzureFileSystem fs, String path)
+ throws IOException {
int index = path.lastIndexOf('/');
String parent = (index == 0) ? "/" : path.substring(0, index);
authorizer.deleteAllAuthRules();
- authorizer.addAuthRule(parent, WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule((path.endsWith("*") ? path : path+"*"), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRule(parent, WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule((path.endsWith("*") ? path : path+"*"), WRITE,
+ getCurrentUserShortName(), true);
fs.updateWasbAuthorizer(authorizer);
}
@@ -101,6 +112,13 @@ public class TestNativeAzureFileSystemAuthorization
}
/**
+ * get current user short name for user context
+ */
+ protected String getCurrentUserShortName() throws IOException {
+ return UserGroupInformation.getCurrentUser().getShortUserName();
+ }
+
+ /**
* Positive test to verify Create access check.
* The file is created directly under an existing folder.
* No intermediate folders need to be created.
@@ -112,7 +130,7 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -137,7 +155,7 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/testCreateAccessCheckPositive/1/2/3");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -164,7 +182,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("create", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -188,8 +206,8 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -215,7 +233,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("create", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -239,8 +257,8 @@ public class TestNativeAzureFileSystemAuthorization
Path intermediateFolders = new Path(parentDir, "1/2/3/");
Path testPath = new Path(intermediateFolders, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -266,8 +284,8 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("liststatus", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -291,8 +309,10 @@ public class TestNativeAzureFileSystemAuthorization
Path srcPath = new Path(parentDir, "test1.dat");
Path dstPath = new Path(parentDir, "test2.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parentDir */
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true); /* for rename */
+ /* to create parentDir */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ /* for rename */
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -321,8 +341,9 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("rename", srcPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */
- authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false);
+ /* to create parent dir */
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -352,9 +373,9 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("rename", dstPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dir */
- authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(parentDstDir.toString(), WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dir */
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -381,9 +402,9 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDstDir = new Path("/testRenameAccessCheckPositiveDst");
Path dstPath = new Path(parentDstDir, "test2.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); /* to create parent dirs */
- authorizer.addAuthRule(parentSrcDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(parentDstDir.toString(), WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dirs */
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -412,8 +433,8 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/testReadAccessCheckPositive");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
FSDataInputStream inputStream = null;
@@ -453,8 +474,8 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("read", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, false);
fs.updateWasbAuthorizer(authorizer);
FSDataInputStream inputStream = null;
@@ -490,7 +511,7 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/");
Path testPath = new Path(parentDir, "test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -514,7 +535,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("delete", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -523,7 +544,7 @@ public class TestNativeAzureFileSystemAuthorization
/* Remove permissions for delete to force failure */
authorizer.deleteAllAuthRules();
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, false);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -531,7 +552,7 @@ public class TestNativeAzureFileSystemAuthorization
finally {
/* Restore permissions to force a successful delete */
authorizer.deleteAllAuthRules();
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -550,9 +571,9 @@ public class TestNativeAzureFileSystemAuthorization
Path parentDir = new Path("/testDeleteIntermediateFolder");
Path testPath = new Path(parentDir, "1/2/test.dat");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true); // for create and delete
- authorizer.addAuthRule("/testDeleteIntermediateFolder*",
- WasbAuthorizationOperations.WRITE.toString(), true); // for recursive delete
+ authorizer.addAuthRuleForOwner("/", WRITE, true); // for create and delete
+ authorizer.addAuthRuleForOwner("/testDeleteIntermediateFolder*",
+ WRITE, true); // for recursive delete
fs.updateWasbAuthorizer(authorizer);
try {
@@ -568,6 +589,365 @@ public class TestNativeAzureFileSystemAuthorization
}
/**
+ * Test to verify access check failure leaves intermediate folders undeleted.
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteAuthCheckFailureLeavesFilesUndeleted() throws Throwable {
+
+ Path parentDir = new Path("/testDeleteAuthCheckFailureLeavesFilesUndeleted");
+ Path testPath1 = new Path(parentDir, "child1/test.dat");
+ Path testPath2 = new Path(parentDir, "child2/test.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath1);
+ fs.create(testPath2);
+ ContractTestUtils.assertPathExists(fs, "testPath1 was not created", testPath1);
+ ContractTestUtils.assertPathExists(fs, "testPath2 was not created", testPath2);
+
+ // revoke write on one of the child folders
+ authorizer.deleteAllAuthRules();
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted",
+ WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted/child2",
+ WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted/child1",
+ WRITE, false);
+
+ assertFalse(fs.delete(parentDir, true));
+
+ // Assert that only child2 contents are deleted
+ ContractTestUtils.assertPathExists(fs, "child1 is deleted!", testPath1);
+ ContractTestUtils.assertPathDoesNotExist(fs, "child2 exists after deletion!", testPath2);
+ ContractTestUtils.assertPathDoesNotExist(fs, "child2 exists after deletion!",
+ new Path("/testDeleteAuthCheckFailureLeavesFilesUndeleted/childPath2"));
+ ContractTestUtils.assertPathExists(fs, "parentDir is deleted!", parentDir);
+
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Positive test to verify file delete with sticky bit on parent.
+ * @throws Throwable
+ */
+ @Test
+ public void testSingleFileDeleteWithStickyBitPositive() throws Throwable {
+
+ Path parentDir = new Path("/testSingleFileDeleteWithStickyBitPositive");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testSingleFileDeleteWithStickyBitPositive",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath);
+
+ // set stickybit on parent directory
+ fs.setPermission(parentDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ assertTrue(fs.delete(testPath, true));
+ ContractTestUtils.assertPathDoesNotExist(fs,
+ "testPath exists after deletion!", testPath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Negative test to verify file delete fails when sticky bit is set on parent
+ * and non-owner user performs delete
+ * @throws Throwable
+ */
+ @Test
+ public void testSingleFileDeleteWithStickyBitNegative() throws Throwable {
+
+ Path parentDir = new Path("/testSingleFileDeleteWithStickyBitNegative");
+ Path testPath = new Path(parentDir, "test.dat");
+
+ expectedEx.expect(WasbAuthorizationException.class);
+ expectedEx.expectMessage(String.format("%s has sticky bit set. File %s cannot be deleted.",
+ parentDir.toString(), testPath.toString()));
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testSingleFileDeleteWithStickyBitNegative",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath);
+ // set stickybit on parent directory
+ fs.setPermission(parentDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ authorizer.addAuthRule(parentDir.toString(), WRITE,
+ getCurrentUserShortName(), true);
+ fs.delete(testPath, true);
+ return null;
+ }
+ });
+ }
+ finally {
+ ContractTestUtils.assertPathExists(fs, "testPath should not be deleted!", testPath);
+
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Positive test to verify file and folder delete succeeds with stickybit
+ * when the owner of the files deletes the file.
+ * @throws Throwable
+ */
+ @Test
+ public void testRecursiveDeleteSucceedsWithStickybit() throws Throwable {
+
+ Path parentDir = new Path("/testRecursiveDeleteSucceedsWithStickybit");
+ Path testFilePath = new Path(parentDir, "child/test.dat");
+ Path testFolderPath = new Path(parentDir, "child/testDirectory");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testRecursiveDeleteSucceedsWithStickybit*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+ fs.mkdirs(testFolderPath);
+ ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
+ // set stickybit on child directory
+ fs.setPermission(new Path(parentDir, "child"),
+ new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+ // perform delete as owner of the files
+ assertTrue(fs.delete(parentDir, true));
+ ContractTestUtils.assertPathDoesNotExist(fs, "parentDir exists after deletion!", parentDir);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test to verify delete fails for child files and folders when
+ * non-owner user performs delete and stickybit is set on parent
+ * @throws Throwable
+ */
+ @Test
+ public void testRecursiveDeleteFailsWithStickybit() throws Throwable {
+
+ Path parentDir = new Path("/testRecursiveDeleteFailsWithStickybit");
+ Path testFilePath = new Path(parentDir, "child/test.dat");
+ Path testFolderPath = new Path(parentDir, "child/testDirectory");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testRecursiveDeleteFailsWithStickybit*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+ fs.mkdirs(testFolderPath);
+ ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
+
+ // set stickybit on child directory
+ fs.setPermission(new Path(parentDir, "child"),
+ new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // Add auth rules for dummyuser
+ authorizer.addAuthRule("/", WRITE,
+ getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/testRecursiveDeleteFailsWithStickybit*",
+ WRITE, getCurrentUserShortName(), true);
+
+ assertFalse(fs.delete(parentDir, true));
+ return null;
+ }
+ });
+
+ ContractTestUtils.assertPathExists(fs, "parentDir is deleted!", parentDir);
+ ContractTestUtils.assertPathExists(fs, "file is deleted!", testFilePath);
+ ContractTestUtils.assertPathExists(fs, "folder is deleted!", testFolderPath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test delete scenario where sticky bit check leaves files/folders not owned
+ * by a specific user intact and the files owned by him/her are deleted
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet()
+ throws Throwable {
+
+ Path parentDir = new Path("/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet");
+ Path testFilePath = new Path(parentDir, "test.dat");
+ Path testFolderPath = new Path(parentDir, "testDirectory");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(
+ "/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+
+ fs.setPermission(parentDir, new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyuser", new String[] {"dummygroup"});
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ authorizer.addAuthRule("/", WRITE,
+ getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
+ WRITE, getCurrentUserShortName(), true);
+
+ fs.create(testFolderPath); // the folder will have owner as dummyuser
+ ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
+ assertFalse(fs.delete(parentDir, true));
+
+ ContractTestUtils.assertPathDoesNotExist(fs, "folder should have been deleted!",
+ testFolderPath);
+ ContractTestUtils.assertPathExists(fs, "parentDir is deleted!", parentDir);
+ ContractTestUtils.assertPathExists(fs, "file is deleted!", testFilePath);
+ return null;
+ }
+ });
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test delete scenario where sticky bit is set and the owner of parent
+ * directory can delete child files/folders which he does not own.
+ * This is according to the sticky bit behaviour specified in hdfs permission
+ * guide which is as follows - The sticky bit can be set on directories,
+ * preventing anyone except the superuser, directory owner or file owner
+ * from deleting or moving the files within the directory
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit() throws Throwable {
+
+ Path parentDir = new Path("/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit");
+ Path testFilePath = new Path(parentDir, "test.dat");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(
+ "/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ // create folder with owner as current user
+ fs.mkdirs(parentDir);
+ ContractTestUtils.assertPathExists(fs, "folder was not created", parentDir);
+
+ // create child with owner as dummyUser
+ UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
+ "dummyUser", new String[] {"dummygroup"});
+ dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ authorizer.addAuthRule("/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit",
+ WRITE, getCurrentUserShortName(), true);
+ fs.create(testFilePath);
+ ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
+
+ fs.setPermission(parentDir,
+ new FsPermission(STICKYBIT_PERMISSION_CONSTANT));
+ return null;
+ }
+ });
+
+ // invoke delete as current user
+ assertTrue(fs.delete(parentDir, true));
+ ContractTestUtils.assertPathDoesNotExist(fs, "parentDir is not deleted!", parentDir);
+ ContractTestUtils.assertPathDoesNotExist(fs, "file is not deleted!", testFilePath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test to verify delete of root succeeds with proper permissions and
+ * leaves root after delete.
+ * @throws Throwable
+ */
+ @Test
+ public void testDeleteScenarioForRoot() throws Throwable {
+ Path rootPath = new Path("/");
+ Path parentDir = new Path("/testDeleteScenarioForRoot");
+ Path testPath1 = new Path(parentDir, "child1/test.dat");
+ Path testPath2 = new Path(parentDir, "child2/testFolder");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/testDeleteScenarioForRoot*",
+ WRITE, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(testPath1);
+ fs.create(testPath2);
+ ContractTestUtils.assertPathExists(fs, "testPath1 was not created", testPath1);
+ ContractTestUtils.assertPathExists(fs, "testPath2 was not created", testPath2);
+
+ assertFalse(fs.delete(rootPath, true));
+
+ ContractTestUtils.assertPathDoesNotExist(fs, "file exists after deletion!", testPath1);
+ ContractTestUtils.assertPathDoesNotExist(fs, "folder exists after deletion!", testPath2);
+ ContractTestUtils.assertPathDoesNotExist(fs, "parentDir exists after deletion!", parentDir);
+ ContractTestUtils.assertPathExists(fs, "Root should not have been deleted!", rootPath);
+ }
+ finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
* Positive test for getFileStatus. No permissions are required for getting filestatus.
* @throws Throwable
*/
@@ -587,7 +967,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsAccessCheckPositive/1/2/3");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -609,7 +989,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsWithExistingHierarchyCheckPositive1");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -636,11 +1016,11 @@ public class TestNativeAzureFileSystemAuthorization
Path childPath2 = new Path(childPath1, "2");
Path childPath3 = new Path(childPath2, "3");
- authorizer.addAuthRule("/",
- WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/",
+ WRITE, true);
- authorizer.addAuthRule(childPath1.toString(),
- WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner(childPath1.toString(),
+ WRITE, true);
fs.updateWasbAuthorizer(authorizer);
@@ -675,7 +1055,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("mkdirs", testPath);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), false);
+ authorizer.addAuthRuleForOwner("/", WRITE, false);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -697,13 +1077,96 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/");
- authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
Path testPathWithTripleSlash = new Path("wasb:///" + testPath);
fs.listStatus(testPathWithTripleSlash);
}
+ /**
+ * Test case when owner matches current user
+ */
+ @Test
+ public void testOwnerPermissionPositive() throws Throwable {
+
+ Path parentDir = new Path("/testOwnerPermissionPositive");
+ Path testPath = new Path(parentDir, "test.data");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+ // additional rule used for assertPathExists
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ // creates parentDir with owner as current user
+ fs.mkdirs(parentDir);
+ ContractTestUtils.assertPathExists(fs, "parentDir does not exist", parentDir);
+
+ fs.create(testPath);
+ fs.getFileStatus(testPath);
+ ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath);
+
+ fs.delete(parentDir, true);
+ ContractTestUtils.assertPathDoesNotExist(fs, "testPath does not exist", testPath);
+
+ } finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Negative test case for owner does not match current user
+ */
+ @Test
+ public void testOwnerPermissionNegative() throws Throwable {
+
+ Path parentDir = new Path("/testOwnerPermissionNegative");
+ Path childDir = new Path(parentDir, "childDir");
+
+ setExpectedFailureMessage("mkdirs", childDir);
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+
+ fs.updateWasbAuthorizer(authorizer);
+
+ try{
+ fs.mkdirs(parentDir);
+ UserGroupInformation ugiSuperUser = UserGroupInformation.createUserForTesting(
+ "testuser", new String[] {});
+
+ ugiSuperUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ fs.mkdirs(childDir);
+ return null;
+ }
+ });
+
+ } finally {
+ allowRecursiveDelete(fs, parentDir.toString());
+ fs.delete(parentDir, true);
+ }
+ }
+
+ /**
+ * Test to verify that retrieving owner information does not
+ * throw when file/folder does not exist
+ */
+ @Test
+ public void testRetrievingOwnerDoesNotFailWhenFileDoesNotExist()
+ throws Throwable {
+
+ Path testdirectory = new Path("/testDirectory123454565");
+
+ String owner = fs.getOwnerForPath(testdirectory);
+ assertEquals("", owner);
+ }
+
/**
* Negative test for setOwner when Authorization is enabled.
*/
@@ -714,7 +1177,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerNegative");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String owner = null;
@@ -748,7 +1211,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerPositive");
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String newOwner = "newowner";
@@ -793,7 +1256,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerPositiveWildcard");
authorizer.init(conf);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String newOwner = "newowner";
@@ -839,7 +1302,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerFailsForIllegalSetup");
authorizer.init(conf);
- authorizer.addAuthRule("/", WasbAuthorizationOperations.WRITE.toString(), true);
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
fs.updateWasbAuthorizer(authorizer);
String owner = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-3661. Basic Federation UI.
(Contributed by Inigo Goiri via curino)
Posted by kk...@apache.org.
YARN-3661. Basic Federation UI. (Contributed by Inigo Goiri via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ceca9694
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ceca9694
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ceca9694
Branch: refs/heads/YARN-6592
Commit: ceca9694f9a0c78d07cab2c382036f175183e67b
Parents: 5a68c2c
Author: Carlo Curino <cu...@apache.org>
Authored: Mon Oct 2 13:03:32 2017 -0700
Committer: Carlo Curino <cu...@apache.org>
Committed: Mon Oct 2 13:03:32 2017 -0700
----------------------------------------------------------------------
.../yarn/server/router/webapp/AboutBlock.java | 91 ++++++++++
.../yarn/server/router/webapp/AboutPage.java | 37 ++++
.../yarn/server/router/webapp/AppsBlock.java | 130 ++++++++++++++
.../yarn/server/router/webapp/AppsPage.java | 77 ++++++++
.../server/router/webapp/FederationBlock.java | 176 +++++++++++++++++++
.../server/router/webapp/FederationPage.java | 57 ++++++
.../yarn/server/router/webapp/NavBlock.java | 46 +++++
.../yarn/server/router/webapp/NodesBlock.java | 109 ++++++++++++
.../yarn/server/router/webapp/NodesPage.java | 60 +++++++
.../server/router/webapp/RouterController.java | 60 +++++++
.../yarn/server/router/webapp/RouterView.java | 52 ++++++
.../yarn/server/router/webapp/RouterWebApp.java | 6 +
.../src/site/markdown/Federation.md | 6 +-
13 files changed, 905 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
new file mode 100644
index 0000000..cd588fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
@@ -0,0 +1,91 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * About block for the Router Web UI.
+ */
+public class AboutBlock extends HtmlBlock {
+
+ private static final long BYTES_IN_MB = 1024 * 1024;
+
+ private final Router router;
+
+ @Inject
+ AboutBlock(Router router, ViewContext ctx) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ protected void render(Block html) {
+ Configuration conf = this.router.getConfig();
+ String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+
+ ClusterMetricsInfo metrics = RouterWebServiceUtil.genericForward(
+ webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null);
+ boolean isEnabled = conf.getBoolean(
+ YarnConfiguration.FEDERATION_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ info("Cluster Status").
+ __("Federation Enabled", isEnabled).
+ __("Applications Submitted", "N/A").
+ __("Applications Pending", "N/A").
+ __("Applications Running", "N/A").
+ __("Applications Failed", "N/A").
+ __("Applications Killed", "N/A").
+ __("Applications Completed", "N/A").
+ __("Containers Allocated", metrics.getContainersAllocated()).
+ __("Containers Reserved", metrics.getReservedContainers()).
+ __("Containers Pending", metrics.getPendingContainers()).
+ __("Available Memory",
+ StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)).
+ __("Allocated Memory",
+ StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)).
+ __("Reserved Memory",
+ StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)).
+ __("Total Memory",
+ StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)).
+ __("Available VirtualCores", metrics.getAvailableVirtualCores()).
+ __("Allocated VirtualCores", metrics.getAllocatedVirtualCores()).
+ __("Reserved VirtualCores", metrics.getReservedVirtualCores()).
+ __("Total VirtualCores", metrics.getTotalVirtualCores()).
+ __("Active Nodes", metrics.getActiveNodes()).
+ __("Lost Nodes", metrics.getLostNodes()).
+ __("Available Nodes", metrics.getDecommissionedNodes()).
+ __("Unhealthy Nodes", metrics.getUnhealthyNodes()).
+ __("Rebooted Nodes", metrics.getRebootedNodes()).
+ __("Total Nodes", metrics.getTotalNodes());
+
+ html.__(InfoBlock.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java
new file mode 100644
index 0000000..3c9f00d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutPage.java
@@ -0,0 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+/**
+ * About page for the Router Web UI.
+ */
+public class AboutPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return AboutBlock.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
new file mode 100644
index 0000000..56fa73e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
+import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * Applications block for the Router Web UI.
+ */
+public class AppsBlock extends HtmlBlock {
+ private final Router router;
+
+ @Inject
+ AppsBlock(Router router, ViewContext ctx) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ protected void render(Block html) {
+ // Get the applications from the Resource Managers
+ Configuration conf = this.router.getConfig();
+ String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+ AppsInfo apps = RouterWebServiceUtil.genericForward(webAppAddress, null,
+ AppsInfo.class, HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null);
+
+ setTitle("Applications");
+
+ TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
+ .tr()
+ .th(".id", "ID")
+ .th(".user", "User")
+ .th(".name", "Name")
+ .th(".type", "Application Type")
+ .th(".queue", "Queue")
+ .th(".priority", "Application Priority")
+ .th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime")
+ .th(".state", "State")
+ .th(".finalstatus", "FinalStatus")
+ .th(".progress", "Progress")
+ .th(".ui", "Tracking UI")
+ .__().__().tbody();
+
+ // Render the applications
+ StringBuilder appsTableData = new StringBuilder("[\n");
+ for (AppInfo app : apps.getApps()) {
+ try {
+
+ String percent = String.format("%.1f", app.getProgress() * 100.0F);
+ String trackingURL =
+ app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
+ // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+ appsTableData.append("[\"")
+ .append("<a href='").append(trackingURL).append("'>")
+ .append(app.getAppId()).append("</a>\",\"")
+ .append(escape(app.getUser())).append("\",\"")
+ .append(escape(app.getName())).append("\",\"")
+ .append(escape(app.getApplicationType())).append("\",\"")
+ .append(escape(app.getQueue())).append("\",\"")
+ .append(String.valueOf(app.getPriority())).append("\",\"")
+ .append(app.getStartTime()).append("\",\"")
+ .append(app.getFinishTime()).append("\",\"")
+ .append(app.getState()).append("\",\"")
+ .append(app.getFinalStatus()).append("\",\"")
+ // Progress bar
+ .append("<br title='").append(percent).append("'> <div class='")
+ .append(C_PROGRESSBAR).append("' title='")
+ .append(join(percent, '%')).append("'> ").append("<div class='")
+ .append(C_PROGRESSBAR_VALUE).append("' style='")
+ .append(join("width:", percent, '%')).append("'> </div> </div>")
+ // History link
+ .append("\",\"<a href='").append(trackingURL).append("'>")
+ .append("History").append("</a>");
+ appsTableData.append("\"],\n");
+
+ } catch (Exception e) {
+ LOG.info(
+ "Cannot add application {}: {}", app.getAppId(), e.getMessage());
+ }
+ }
+ if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
+ appsTableData.delete(appsTableData.length() - 2,
+ appsTableData.length() - 1);
+ }
+ appsTableData.append("]");
+ html.script().$type("text/javascript")
+ .__("var appsTableData=" + appsTableData).__();
+
+ tbody.__().__();
+ }
+
+ private static String escape(String str) {
+ return escapeJavaScript(escapeHtml(str));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
new file mode 100644
index 0000000..12d0b5b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+class AppsPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ set(DATATABLES_ID, "apps");
+ set(initID(DATATABLES, "apps"), appsTableInit());
+ setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");
+
+ // Set the correct title.
+ String reqState = $(APP_STATE);
+ reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
+ setTitle(sjoin(reqState, "Applications"));
+ }
+
+ private String appsTableInit() {
+ // id, user, name, queue, starttime, finishtime, state, status, progress, ui
+ return tableInit()
+ .append(", 'aaData': appsTableData")
+ .append(", bDeferRender: true")
+ .append(", bProcessing: true")
+
+ .append("\n, aoColumnDefs: ")
+ .append(getAppsTableColumnDefs())
+
+ // Sort by id upon page load
+ .append(", aaSorting: [[0, 'desc']]}").toString();
+ }
+
+ protected String getAppsTableColumnDefs() {
+ StringBuilder sb = new StringBuilder();
+ return sb
+ .append("[\n")
+ .append("{'sType':'string', 'aTargets': [0]")
+ .append(", 'mRender': parseHadoopID }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [6, 7]")
+ .append(", 'mRender': renderHadoopDate }")
+
+ .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets': [10]")
+ .append(", 'mRender': parseHadoopProgress }]").toString();
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return AppsBlock.class;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
new file mode 100644
index 0000000..d84c705
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+
+class FederationBlock extends HtmlBlock {
+
+ private static final long BYTES_IN_MB = 1024 * 1024;
+
+ private final Router router;
+
+ @Inject
+ FederationBlock(ViewContext ctx, Router router) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ public void render(Block html) {
+ Configuration conf = this.router.getConfig();
+ boolean isEnabled = conf.getBoolean(
+ YarnConfiguration.FEDERATION_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ if (isEnabled) {
+ setTitle("Federation");
+
+ // Table header
+ TBODY<TABLE<Hamlet>> tbody = html.table("#rms").thead().tr()
+ .th(".id", "SubCluster")
+ .th(".submittedA", "Applications Submitted*")
+ .th(".pendingA", "Applications Pending*")
+ .th(".runningA", "Applications Running*")
+ .th(".failedA", "Applications Failed*")
+ .th(".killedA", "Applications Killed*")
+ .th(".completedA", "Applications Completed*")
+ .th(".contAllocated", "Containers Allocated")
+ .th(".contReserved", "Containers Reserved")
+ .th(".contPending", "Containers Pending")
+ .th(".availableM", "Available Memory")
+ .th(".allocatedM", "Allocated Memory")
+ .th(".reservedM", "Reserved Memory")
+ .th(".totalM", "Total Memory")
+ .th(".availableVC", "Available VirtualCores")
+ .th(".allocatedVC", "Allocated VirtualCores")
+ .th(".reservedVC", "Reserved VirtualCores")
+ .th(".totalVC", "Total VirtualCores")
+ .th(".activeN", "Active Nodes")
+ .th(".lostN", "Lost Nodes")
+ .th(".availableN", "Available Nodes")
+ .th(".unhealtyN", "Unhealthy Nodes")
+ .th(".rebootedN", "Rebooted Nodes")
+ .th(".totalN", "Total Nodes")
+ .__().__().tbody();
+
+ try {
+ // Binding to the FederationStateStore
+ FederationStateStoreFacade facade =
+ FederationStateStoreFacade.getInstance();
+ Map<SubClusterId, SubClusterInfo> subClustersInfo =
+ facade.getSubClusters(true);
+
+ // Sort the SubClusters
+ List<SubClusterInfo> subclusters = new ArrayList<>();
+ subclusters.addAll(subClustersInfo.values());
+ Comparator<? super SubClusterInfo> cmp =
+ new Comparator<SubClusterInfo>() {
+ @Override
+ public int compare(SubClusterInfo o1, SubClusterInfo o2) {
+ return o1.getSubClusterId().compareTo(o2.getSubClusterId());
+ }
+ };
+ Collections.sort(subclusters, cmp);
+
+ for (SubClusterInfo subcluster : subclusters) {
+ SubClusterId subClusterId = subcluster.getSubClusterId();
+ String webAppAddress = subcluster.getRMWebServiceAddress();
+ String capability = subcluster.getCapability();
+ ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability);
+
+ // Building row per SubCluster
+ tbody.tr().td().a("//" + webAppAddress, subClusterId.toString()).__()
+ .td(Integer.toString(subClusterInfo.getAppsSubmitted()))
+ .td(Integer.toString(subClusterInfo.getAppsPending()))
+ .td(Integer.toString(subClusterInfo.getAppsRunning()))
+ .td(Integer.toString(subClusterInfo.getAppsFailed()))
+ .td(Integer.toString(subClusterInfo.getAppsKilled()))
+ .td(Integer.toString(subClusterInfo.getAppsCompleted()))
+ .td(Integer.toString(subClusterInfo.getContainersAllocated()))
+ .td(Integer.toString(subClusterInfo.getReservedContainers()))
+ .td(Integer.toString(subClusterInfo.getPendingContainers()))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getAvailableMB() * BYTES_IN_MB))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getAllocatedMB() * BYTES_IN_MB))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getReservedMB() * BYTES_IN_MB))
+ .td(StringUtils.byteDesc(
+ subClusterInfo.getTotalMB() * BYTES_IN_MB))
+ .td(Long.toString(subClusterInfo.getAvailableVirtualCores()))
+ .td(Long.toString(subClusterInfo.getAllocatedVirtualCores()))
+ .td(Long.toString(subClusterInfo.getReservedVirtualCores()))
+ .td(Long.toString(subClusterInfo.getTotalVirtualCores()))
+ .td(Integer.toString(subClusterInfo.getActiveNodes()))
+ .td(Integer.toString(subClusterInfo.getLostNodes()))
+ .td(Integer.toString(subClusterInfo.getDecommissionedNodes()))
+ .td(Integer.toString(subClusterInfo.getUnhealthyNodes()))
+ .td(Integer.toString(subClusterInfo.getRebootedNodes()))
+ .td(Integer.toString(subClusterInfo.getTotalNodes())).__();
+ }
+ } catch (YarnException e) {
+ LOG.error("Cannot render ResourceManager", e);
+ }
+
+ tbody.__().__().div()
+ .p().__("*The application counts are local per subcluster").__().__();
+ } else {
+ setTitle("Federation is not Enabled!");
+ }
+ }
+
+ private static ClusterMetricsInfo getClusterMetricsInfo(String capability) {
+ ClusterMetricsInfo clusterMetrics = null;
+ try {
+ JSONJAXBContext jc = new JSONJAXBContext(
+ JSONConfiguration.mapped().rootUnwrapping(false).build(),
+ ClusterMetricsInfo.class);
+ JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller();
+ clusterMetrics = unmarshaller.unmarshalFromJSON(
+ new StringReader(capability), ClusterMetricsInfo.class);
+ } catch (Exception e) {
+ LOG.error("Cannot parse SubCluster info", e);
+ }
+ return clusterMetrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
new file mode 100644
index 0000000..ddd2866
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
@@ -0,0 +1,57 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+/**
+ * Renders a block for the applications with metrics information.
+ */
+class FederationPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ setTitle("Federation");
+ set(DATATABLES_ID, "rms");
+ set(initID(DATATABLES, "rms"), rmsTableInit());
+ setTableStyles(html, "rms", ".healthStatus {width:10em}",
+ ".healthReport {width:10em}");
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return FederationBlock.class;
+ }
+
+ private String rmsTableInit() {
+ StringBuilder b = tableInit().append(", aoColumnDefs: [");
+ b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
+ b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
+ + "'aTargets': [ 8, 9 ] }");
+ b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
+ b.append("]}");
+ return b.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
new file mode 100644
index 0000000..9c39bb7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
@@ -0,0 +1,46 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+/**
+ * Navigation block for the Router Web UI.
+ */
+public class NavBlock extends HtmlBlock {
+
+ @Override
+ public void render(Block html) {
+ html.
+ div("#nav").
+ h3("Cluster").
+ ul().
+ li().a(url(""), "About").__().
+ li().a(url("federation"), "Federation").__().
+ li().a(url("nodes"), "Nodes").__().
+ li().a(url("apps"), "Applications").__().
+ __().
+ h3("Tools").
+ ul().
+ li().a("/conf", "Configuration").__().
+ li().a("/logs", "Local logs").__().
+ li().a("/stacks", "Server stacks").__().
+ li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
new file mode 100644
index 0000000..720302e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+
+/**
+ * Nodes block for the Router Web UI.
+ */
+public class NodesBlock extends HtmlBlock {
+
+ private static final long BYTES_IN_MB = 1024 * 1024;
+
+ private final Router router;
+
+ @Inject
+ NodesBlock(Router router, ViewContext ctx) {
+ super(ctx);
+ this.router = router;
+ }
+
+ @Override
+ protected void render(Block html) {
+ // Get the node info from the federation
+ Configuration conf = this.router.getConfig();
+ String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+ NodesInfo nodes = RouterWebServiceUtil.genericForward(webAppAddress, null,
+ NodesInfo.class, HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null);
+
+ setTitle("Nodes");
+
+ TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").thead().tr()
+ .th(".nodelabels", "Node Labels")
+ .th(".rack", "Rack")
+ .th(".state", "Node State")
+ .th(".nodeaddress", "Node Address")
+ .th(".nodehttpaddress", "Node HTTP Address")
+ .th(".lastHealthUpdate", "Last health-update")
+ .th(".healthReport", "Health-report")
+ .th(".containers", "Containers")
+ .th(".mem", "Mem Used")
+ .th(".mem", "Mem Avail")
+ .th(".vcores", "VCores Used")
+ .th(".vcores", "VCores Avail")
+ .th(".nodeManagerVersion", "Version")
+ .__().__().tbody();
+
+ // Add nodes to the web UI
+ for (NodeInfo info : nodes.getNodes()) {
+ int usedMemory = (int) info.getUsedMemory();
+ int availableMemory = (int) info.getAvailableMemory();
+ TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
+ row.td().__(StringUtils.join(",", info.getNodeLabels())).__();
+ row.td().__(info.getRack()).__();
+ row.td().__(info.getState()).__();
+ row.td().__(info.getNodeId()).__();
+ boolean isInactive = false;
+ if (isInactive) {
+ row.td().__("N/A").__();
+ } else {
+ String httpAddress = info.getNodeHTTPAddress();
+ row.td().a("//" + httpAddress, httpAddress).__();
+ }
+ row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__()
+ .__(Times.format(info.getLastHealthUpdate())).__()
+ .td(info.getHealthReport())
+ .td(String.valueOf(info.getNumContainers())).td().br()
+ .$title(String.valueOf(usedMemory)).__()
+ .__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br()
+ .$title(String.valueOf(availableMemory)).__()
+ .__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__()
+ .td(String.valueOf(info.getUsedVirtualCores()))
+ .td(String.valueOf(info.getAvailableVirtualCores()))
+ .td(info.getVersion()).__();
+ }
+ tbody.__().__();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
new file mode 100644
index 0000000..0351cba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+
+class NodesPage extends RouterView {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+ String type = $(NODE_STATE);
+ String title = "Nodes of the cluster";
+ if (type != null && !type.isEmpty()) {
+ title = title + " (" + type + ")";
+ }
+ setTitle(title);
+ set(DATATABLES_ID, "nodes");
+ set(initID(DATATABLES, "nodes"), nodesTableInit());
+ setTableStyles(html, "nodes", ".healthStatus {width:10em}",
+ ".healthReport {width:10em}");
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return NodesBlock.class;
+ }
+
+ private String nodesTableInit() {
+ StringBuilder b = tableInit().append(", aoColumnDefs: [");
+ b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
+ b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
+ + "'aTargets': [ 2, 3, 4, 5, 6 ] }");
+ b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
+ b.append("]}");
+ return b.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
new file mode 100644
index 0000000..e8cd0de
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
@@ -0,0 +1,60 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.Controller;
+
+import com.google.inject.Inject;
+
+/**
+ * Controller for the Router Web UI.
+ */
+public class RouterController extends Controller {
+
+ @Inject
+ RouterController(RequestContext ctx) {
+ super(ctx);
+ }
+
+ @Override
+ public void index() {
+ setTitle("Router");
+ render(AboutPage.class);
+ }
+
+ public void about() {
+ setTitle("About the Cluster");
+ render(AboutPage.class);
+ }
+
+ public void federation() {
+ setTitle("Federation");
+ render(FederationPage.class);
+ }
+
+ public void apps() {
+ setTitle("Applications");
+ render(AppsPage.class);
+ }
+
+ public void nodes() {
+ setTitle("Nodes");
+ render(NodesPage.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java
new file mode 100644
index 0000000..b377f7d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterView.java
@@ -0,0 +1,52 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+
+/**
+ * View for the Router Web UI.
+ */
+public class RouterView extends TwoColumnLayout {
+
+ @Override
+ protected void preHead(Page.HTML<__> html) {
+ commonPreHead(html);
+
+ setTitle("Router");
+ }
+
+ protected void commonPreHead(Page.HTML<__> html) {
+ set(ACCORDION_ID, "nav");
+ set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
+ }
+
+ @Override
+ protected Class<? extends SubView> nav() {
+ return NavBlock.class;
+ }
+
+ @Override
+ protected Class<? extends SubView> content() {
+ return AboutBlock.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
index 5436bad..ba07a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
@@ -44,5 +44,11 @@ public class RouterWebApp extends WebApp implements YarnWebParams {
if (router != null) {
bind(Router.class).toInstance(router);
}
+ route("/", RouterController.class);
+ route("/cluster", RouterController.class, "about");
+ route("/about", RouterController.class, "about");
+ route("/apps", RouterController.class, "apps");
+ route("/nodes", RouterController.class, "nodes");
+ route("/federation", RouterController.class, "federation");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ceca9694/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index 8a6c137..ef0f713 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -59,6 +59,7 @@ URL and redirects the application submission request to the appropriate sub-clus
The Router exposes the ApplicationClientProtocol to the outside world, transparently hiding the presence of multiple RMs. To achieve this the Router also persists the mapping
between the application and its home sub-cluster into the State Store. This allows Routers to be soft-state while supporting user requests cheaply, as any Router can recover
this application to home sub-cluster mapping and direct requests to the right RM without broadcasting them. For performance caching and session stickiness might be advisable.
+The state of the federation (including applications and nodes) is exposed through the Web UI.
###AMRMProxy
The AMRMProxy is a key component to allow the application to scale and run across sub-clusters. The AMRMProxy runs on all the NM machines and acts as a proxy to the
@@ -246,9 +247,9 @@ Optional:
|:---- |:---- |
|`yarn.router.hostname` | `0.0.0.0` | Router host name.
|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. |
-|`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. |
+|`yarn.router.webapp.address` | `0.0.0.0:8089` | Webapp address at the router. |
|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. |
-|`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. |
+|`yarn.router.webapp.https.address` | `0.0.0.0:8091` | Secure webapp address at the router. |
|`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. |
|`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. |
|`yarn.federation.cache-ttl.secs` | `60` | The Router caches informations, and this is the time to leave before the cache is invalidated. |
@@ -306,4 +307,5 @@ The output from this particular example job should be something like:
Job Finished in 30.586 seconds
Estimated value of Pi is 3.14250000......
+The state of the job can also be tracked on the Router Web UI at `routerhost:8089`.
Note that no change in the code or recompilation of the input jar was required to use federation. Also, the output of this job is the exact same as it would be when run without federation. Also, in order to get the full benefit of federation, use a large enough number of mappers such that more than one cluster is required. That number happens to be 16 in the case of the above example.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-7259. Add size-based rolling
policy to LogAggregationIndexedFileController. (xgong via wangda)
Posted by kk...@apache.org.
YARN-7259. Add size-based rolling policy to LogAggregationIndexedFileController. (xgong via wangda)
Change-Id: Ifaf82c0aee6b73b9b6ebf103aa72e131e3942f31
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/280080fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/280080fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/280080fa
Branch: refs/heads/YARN-6592
Commit: 280080fad01304c85a9ede4d4f7b707eb36c0155
Parents: 2c62ff7
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 2 15:30:22 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Oct 2 15:30:22 2017 -0700
----------------------------------------------------------------------
.../ifile/IndexedFileAggregatedLogsBlock.java | 14 +-
.../LogAggregationIndexedFileController.java | 397 +++++++++++++------
.../TestLogAggregationIndexFileController.java | 67 +++-
3 files changed, 340 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/280080fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
index c4cbfda..5439b53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
@@ -101,10 +101,9 @@ public class IndexedFileAggregatedLogsBlock extends LogAggregationHtmlBlock {
return;
}
- Map<String, FileStatus> checkSumFiles;
+ Map<String, Long> checkSumFiles;
try {
- checkSumFiles = fileController.filterFiles(nodeFiles,
- LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX);
+ checkSumFiles = fileController.parseCheckSumFiles(nodeFiles);
} catch (IOException ex) {
LOG.error("Error getting logs for " + logEntity, ex);
html.h1("Error getting logs for " + logEntity);
@@ -125,12 +124,11 @@ public class IndexedFileAggregatedLogsBlock extends LogAggregationHtmlBlock {
String desiredLogType = $(CONTAINER_LOG_TYPE);
try {
for (FileStatus thisNodeFile : fileToRead) {
- FileStatus checkSum = fileController.getAllChecksumFiles(
- checkSumFiles, thisNodeFile.getPath().getName());
+ Long checkSumIndex = checkSumFiles.get(
+ thisNodeFile.getPath().getName());
long endIndex = -1;
- if (checkSum != null) {
- endIndex = fileController.loadIndexedLogsCheckSum(
- checkSum.getPath());
+ if (checkSumIndex != null) {
+ endIndex = checkSumIndex.longValue();
}
IndexedLogsMeta indexedLogsMeta = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/280080fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
index 243945e..800c0a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
@@ -29,6 +29,8 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
@@ -41,7 +43,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.UUID;
import org.apache.commons.lang.SerializationUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -54,6 +55,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.HarFs;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
@@ -77,6 +79,8 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.View.ViewContext;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block;
@@ -102,7 +106,8 @@ public class LogAggregationIndexedFileController
"indexedFile.fs.op.num-retries";
private static final String FS_RETRY_INTERVAL_MS_ATTR =
"indexedFile.fs.retry-interval-ms";
- private static final int UUID_LENGTH = 36;
+ private static final String LOG_ROLL_OVER_MAX_FILE_SIZE_GB =
+ "indexedFile.log.roll-over.max-file-size-gb";
@VisibleForTesting
public static final String CHECK_SUM_FILE_SUFFIX = "-checksum";
@@ -121,7 +126,10 @@ public class LogAggregationIndexedFileController
private Path remoteLogCheckSumFile;
private FileContext fc;
private UserGroupInformation ugi;
- private String uuid = null;
+ private byte[] uuid = null;
+ private final int UUID_LENGTH = 32;
+ private long logRollOverMaxFileSize;
+ private Clock sysClock;
public LogAggregationIndexedFileController() {}
@@ -164,6 +172,8 @@ public class LogAggregationIndexedFileController
compressName);
this.fsNumRetries = conf.getInt(FS_NUM_RETRIES_ATTR, 3);
this.fsRetryInterval = conf.getLong(FS_RETRY_INTERVAL_MS_ATTR, 1000L);
+ this.logRollOverMaxFileSize = getRollOverLogMaxSize(conf);
+ this.sysClock = getSystemClock();
}
@Override
@@ -173,11 +183,12 @@ public class LogAggregationIndexedFileController
final UserGroupInformation userUgi = context.getUserUgi();
final Map<ApplicationAccessType, String> appAcls = context.getAppAcls();
final String nodeId = context.getNodeId().toString();
+ final ApplicationId appId = context.getAppId();
final Path remoteLogFile = context.getRemoteNodeLogFileForApp();
this.ugi = userUgi;
logAggregationSuccessfullyInThisCyCle = false;
logsMetaInThisCycle = new IndexedPerAggregationLogMeta();
- logAggregationTimeInThisCycle = System.currentTimeMillis();
+ logAggregationTimeInThisCycle = this.sysClock.getTime();
logsMetaInThisCycle.setUploadTimeStamp(logAggregationTimeInThisCycle);
logsMetaInThisCycle.setRemoteNodeFile(remoteLogFile.getName());
try {
@@ -187,57 +198,6 @@ public class LogAggregationIndexedFileController
fc = FileContext.getFileContext(
remoteRootLogDir.toUri(), conf);
fc.setUMask(APP_LOG_FILE_UMASK);
- boolean fileExist = fc.util().exists(remoteLogFile);
- if (fileExist && context.isLogAggregationInRolling()) {
- fsDataOStream = fc.create(remoteLogFile,
- EnumSet.of(CreateFlag.APPEND),
- new Options.CreateOpts[] {});
- if (uuid == null) {
- FSDataInputStream fsDataInputStream = null;
- try {
- fsDataInputStream = fc.open(remoteLogFile);
- byte[] b = new byte[UUID_LENGTH];
- int actual = fsDataInputStream.read(b);
- if (actual != UUID_LENGTH) {
- // Get an error when parse the UUID from existed log file.
- // Simply OverWrite the existed log file and re-create the
- // UUID.
- fsDataOStream = fc.create(remoteLogFile,
- EnumSet.of(CreateFlag.OVERWRITE),
- new Options.CreateOpts[] {});
- uuid = UUID.randomUUID().toString();
- fsDataOStream.write(uuid.getBytes(Charset.forName("UTF-8")));
- fsDataOStream.flush();
- } else {
- uuid = new String(b, Charset.forName("UTF-8"));
- }
- } finally {
- IOUtils.cleanupWithLogger(LOG, fsDataInputStream);
- }
- }
- // if the remote log file exists, but we do not have any
- // indexedLogsMeta. We need to re-load indexedLogsMeta from
- // the existing remote log file. If the re-load fails, we simply
- // re-create a new indexedLogsMeta object. And will re-load
- // the indexedLogsMeta from checksum file later.
- if (indexedLogsMeta == null) {
- try {
- indexedLogsMeta = loadIndexedLogsMeta(remoteLogFile);
- } catch (IOException ex) {
- // DO NOTHING
- }
- }
- } else {
- fsDataOStream = fc.create(remoteLogFile,
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
- new Options.CreateOpts[] {});
- if (uuid == null) {
- uuid = UUID.randomUUID().toString();
- }
- byte[] b = uuid.getBytes(Charset.forName("UTF-8"));
- fsDataOStream.write(b);
- fsDataOStream.flush();
- }
if (indexedLogsMeta == null) {
indexedLogsMeta = new IndexedLogsMeta();
indexedLogsMeta.setVersion(VERSION);
@@ -249,44 +209,24 @@ public class LogAggregationIndexedFileController
YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE);
indexedLogsMeta.setCompressName(compressName);
}
- final long currentAggregatedLogFileLength = fc
- .getFileStatus(remoteLogFile).getLen();
- // only check the check-sum file when we are in append mode
+ Path aggregatedLogFile = null;
if (context.isLogAggregationInRolling()) {
- // check whether the checksum file exists to figure out
- // whether the previous log aggregation process is successful
- // and the aggregated log file is corrupted or not.
- remoteLogCheckSumFile = new Path(remoteLogFile.getParent(),
- (remoteLogFile.getName() + CHECK_SUM_FILE_SUFFIX));
- boolean exist = fc.util().exists(remoteLogCheckSumFile);
- if (!exist) {
- FSDataOutputStream checksumFileOutputStream = null;
- try {
- checksumFileOutputStream = fc.create(remoteLogCheckSumFile,
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
- new Options.CreateOpts[] {});
- checksumFileOutputStream.writeLong(
- currentAggregatedLogFileLength);
- } finally {
- IOUtils.cleanupWithLogger(LOG, checksumFileOutputStream);
- }
- } else {
- FSDataInputStream checksumFileInputStream = null;
- try {
- checksumFileInputStream = fc.open(remoteLogCheckSumFile);
- long endIndex = checksumFileInputStream.readLong();
- IndexedLogsMeta recoveredLogsMeta = loadIndexedLogsMeta(
- remoteLogFile, endIndex);
- if (recoveredLogsMeta == null) {
- indexedLogsMeta.getLogMetas().clear();
- } else {
- indexedLogsMeta = recoveredLogsMeta;
- }
- } finally {
- IOUtils.cleanupWithLogger(LOG, checksumFileInputStream);
- }
+ aggregatedLogFile = initializeWriterInRolling(
+ remoteLogFile, appId, nodeId);
+ } else {
+ aggregatedLogFile = remoteLogFile;
+ fsDataOStream = fc.create(remoteLogFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
+ new Options.CreateOpts[] {});
+ if (uuid == null) {
+ uuid = createUUID(appId);
}
+ fsDataOStream.write(uuid);
+ fsDataOStream.flush();
}
+
+ long aggregatedLogFileLength = fc.getFileStatus(
+ aggregatedLogFile).getLen();
// append a simple character("\n") to move the writer cursor, so
// we could get the correct position when we call
// fsOutputStream.getStartPos()
@@ -294,11 +234,11 @@ public class LogAggregationIndexedFileController
fsDataOStream.write(dummyBytes);
fsDataOStream.flush();
- if (fsDataOStream.getPos() >= (currentAggregatedLogFileLength
+ if (fsDataOStream.getPos() >= (aggregatedLogFileLength
+ dummyBytes.length)) {
currentOffSet = 0;
} else {
- currentOffSet = currentAggregatedLogFileLength;
+ currentOffSet = aggregatedLogFileLength;
}
return null;
}
@@ -308,6 +248,104 @@ public class LogAggregationIndexedFileController
}
}
+ private Path initializeWriterInRolling(final Path remoteLogFile,
+ final ApplicationId appId, final String nodeId) throws Exception {
+ Path aggregatedLogFile = null;
+ // check uuid
+ // if we can not find uuid, we would load the uuid
+ // from previous aggregated log files, and at the same
+ // time, we would delete any aggregated log files which
+ // has invalid uuid.
+ if (uuid == null) {
+ uuid = loadUUIDFromLogFile(fc, remoteLogFile.getParent(),
+ appId, nodeId);
+ }
+ Path currentRemoteLogFile = getCurrentRemoteLogFile(
+ fc, remoteLogFile.getParent(), nodeId);
+ // check checksum file
+ boolean overwriteCheckSum = true;
+ remoteLogCheckSumFile = new Path(remoteLogFile.getParent(),
+ (remoteLogFile.getName() + CHECK_SUM_FILE_SUFFIX));
+ if(fc.util().exists(remoteLogCheckSumFile)) {
+ // if the checksum file exists, we should reset cached
+ // indexedLogsMeta.
+ indexedLogsMeta.getLogMetas().clear();
+ if (currentRemoteLogFile != null) {
+ FSDataInputStream checksumFileInputStream = null;
+ try {
+ checksumFileInputStream = fc.open(remoteLogCheckSumFile);
+ int nameLength = checksumFileInputStream.readInt();
+ byte[] b = new byte[nameLength];
+ int actualLength = checksumFileInputStream.read(b);
+ if (actualLength == nameLength) {
+ String recoveredLogFile = new String(
+ b, Charset.forName("UTF-8"));
+ if (recoveredLogFile.equals(
+ currentRemoteLogFile.getName())) {
+ overwriteCheckSum = false;
+ long endIndex = checksumFileInputStream.readLong();
+ IndexedLogsMeta recoveredLogsMeta = null;
+ try {
+ truncateFileWithRetries(fc, currentRemoteLogFile,
+ endIndex);
+ recoveredLogsMeta = loadIndexedLogsMeta(
+ currentRemoteLogFile);
+ } catch (Exception ex) {
+ recoveredLogsMeta = loadIndexedLogsMeta(
+ currentRemoteLogFile, endIndex);
+ }
+ if (recoveredLogsMeta != null) {
+ indexedLogsMeta = recoveredLogsMeta;
+ }
+ }
+ }
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, checksumFileInputStream);
+ }
+ }
+ }
+ // check whether we need roll over old logs
+ if (currentRemoteLogFile == null || isRollover(
+ fc, currentRemoteLogFile)) {
+ indexedLogsMeta.getLogMetas().clear();
+ overwriteCheckSum = true;
+ aggregatedLogFile = new Path(remoteLogFile.getParent(),
+ remoteLogFile.getName() + "_" + sysClock.getTime());
+ fsDataOStream = fc.create(aggregatedLogFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
+ new Options.CreateOpts[] {});
+ // writes the uuid
+ fsDataOStream.write(uuid);
+ fsDataOStream.flush();
+ } else {
+ aggregatedLogFile = currentRemoteLogFile;
+ fsDataOStream = fc.create(currentRemoteLogFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND),
+ new Options.CreateOpts[] {});
+ }
+ // recreate checksum file if needed before aggregate the logs
+ if (overwriteCheckSum) {
+ final long currentAggregatedLogFileLength = fc
+ .getFileStatus(aggregatedLogFile).getLen();
+ FSDataOutputStream checksumFileOutputStream = null;
+ try {
+ checksumFileOutputStream = fc.create(remoteLogCheckSumFile,
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
+ new Options.CreateOpts[] {});
+ String fileName = aggregatedLogFile.getName();
+ checksumFileOutputStream.writeInt(fileName.length());
+ checksumFileOutputStream.write(fileName.getBytes(
+ Charset.forName("UTF-8")));
+ checksumFileOutputStream.writeLong(
+ currentAggregatedLogFileLength);
+ checksumFileOutputStream.flush();
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, checksumFileOutputStream);
+ }
+ }
+ return aggregatedLogFile;
+ }
+
@Override
public void closeWriter() {
IOUtils.cleanupWithLogger(LOG, this.fsDataOStream);
@@ -390,8 +428,7 @@ public class LogAggregationIndexedFileController
this.fsDataOStream.write(b);
int length = b.length;
this.fsDataOStream.writeInt(length);
- byte[] separator = this.uuid.getBytes(Charset.forName("UTF-8"));
- this.fsDataOStream.write(separator);
+ this.fsDataOStream.write(uuid);
if (logAggregationSuccessfullyInThisCyCle &&
record.isLogAggregationInRolling()) {
deleteFileWithRetries(fc, ugi, remoteLogCheckSumFile);
@@ -410,6 +447,30 @@ public class LogAggregationIndexedFileController
}.runWithRetries();
}
+ private void deleteFileWithRetries(final FileContext fileContext,
+ final Path deletePath) throws Exception {
+ new FSAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ if (fileContext.util().exists(deletePath)) {
+ fileContext.delete(deletePath, false);
+ }
+ return null;
+ }
+ }.runWithRetries();
+ }
+
+ private void truncateFileWithRetries(final FileContext fileContext,
+ final Path truncatePath, final long newLength) throws Exception {
+ new FSAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ fileContext.truncate(truncatePath, newLength);
+ return null;
+ }
+ }.runWithRetries();
+ }
+
private Object deleteFileWithPrivilege(final FileContext fileContext,
final UserGroupInformation userUgi, final Path fileToDelete)
throws Exception {
@@ -449,18 +510,16 @@ public class LogAggregationIndexedFileController
throw new IOException("There is no available log fils for "
+ "application:" + appId);
}
- Map<String, FileStatus> checkSumFiles = filterFiles(
- nodeFiles, CHECK_SUM_FILE_SUFFIX);
+ Map<String, Long> checkSumFiles = parseCheckSumFiles(nodeFiles);
List<FileStatus> fileToRead = getNodeLogFileToRead(
nodeFiles, nodeIdStr, appId);
byte[] buf = new byte[65535];
for (FileStatus thisNodeFile : fileToRead) {
String nodeName = thisNodeFile.getPath().getName();
- FileStatus checkSum = getAllChecksumFiles(checkSumFiles,
- thisNodeFile.getPath().getName());
+ Long checkSumIndex = checkSumFiles.get(nodeName);
long endIndex = -1;
- if (checkSum != null) {
- endIndex = loadIndexedLogsCheckSum(checkSum.getPath());
+ if (checkSumIndex != null) {
+ endIndex = checkSumIndex.longValue();
}
IndexedLogsMeta indexedLogsMeta = null;
try {
@@ -565,17 +624,16 @@ public class LogAggregationIndexedFileController
throw new IOException("There is no available log fils for "
+ "application:" + appId);
}
- Map<String, FileStatus> checkSumFiles = filterFiles(
- nodeFiles, CHECK_SUM_FILE_SUFFIX);
+ Map<String, Long> checkSumFiles = parseCheckSumFiles(nodeFiles);
List<FileStatus> fileToRead = getNodeLogFileToRead(
nodeFiles, nodeIdStr, appId);
for(FileStatus thisNodeFile : fileToRead) {
try {
- FileStatus checkSum = getAllChecksumFiles(checkSumFiles,
+ Long checkSumIndex = checkSumFiles.get(
thisNodeFile.getPath().getName());
long endIndex = -1;
- if (checkSum != null) {
- endIndex = loadIndexedLogsCheckSum(checkSum.getPath());
+ if (checkSumIndex != null) {
+ endIndex = checkSumIndex.longValue();
}
IndexedLogsMeta current = loadIndexedLogsMeta(
thisNodeFile.getPath(), endIndex);
@@ -627,21 +685,46 @@ public class LogAggregationIndexedFileController
}
@Private
- public Map<String, FileStatus> filterFiles(
- List<FileStatus> fileList, final String suffix) throws IOException {
- Map<String, FileStatus> checkSumFiles = new HashMap<>();
+ public Map<String, Long> parseCheckSumFiles(
+ List<FileStatus> fileList) throws IOException {
+ Map<String, Long> checkSumFiles = new HashMap<>();
Set<FileStatus> status = new HashSet<FileStatus>(fileList);
Iterable<FileStatus> mask =
Iterables.filter(status, new Predicate<FileStatus>() {
@Override
public boolean apply(FileStatus next) {
return next.getPath().getName().endsWith(
- suffix);
+ CHECK_SUM_FILE_SUFFIX);
}
});
status = Sets.newHashSet(mask);
+ FileContext fc = null;
for (FileStatus file : status) {
- checkSumFiles.put(file.getPath().getName(), file);
+ FSDataInputStream checksumFileInputStream = null;
+ try {
+ if (fc == null) {
+ fc = FileContext.getFileContext(file.getPath().toUri(), conf);
+ }
+ String nodeName = null;
+ long index = 0L;
+ checksumFileInputStream = fc.open(file.getPath());
+ int nameLength = checksumFileInputStream.readInt();
+ byte[] b = new byte[nameLength];
+ int actualLength = checksumFileInputStream.read(b);
+ if (actualLength == nameLength) {
+ nodeName = new String(b, Charset.forName("UTF-8"));
+ index = checksumFileInputStream.readLong();
+ } else {
+ continue;
+ }
+ if (nodeName != null && !nodeName.isEmpty()) {
+ checkSumFiles.put(nodeName, Long.valueOf(index));
+ }
+ } catch (IOException ex) {
+ continue;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, checksumFileInputStream);
+ }
}
return checkSumFiles;
}
@@ -755,20 +838,6 @@ public class LogAggregationIndexedFileController
return loadIndexedLogsMeta(remoteLogPath, -1);
}
- @Private
- public long loadIndexedLogsCheckSum(Path remoteLogCheckSumPath)
- throws IOException {
- FileContext fileContext =
- FileContext.getFileContext(remoteLogCheckSumPath.toUri(), conf);
- FSDataInputStream fsDataIStream = null;
- try {
- fsDataIStream = fileContext.open(remoteLogCheckSumPath);
- return fsDataIStream.readLong();
- } finally {
- IOUtils.cleanupWithLogger(LOG, fsDataIStream);
- }
- }
-
/**
* This IndexedLogsMeta includes all the meta information
* for the aggregated log file.
@@ -1034,6 +1103,13 @@ public class LogAggregationIndexedFileController
return conf.getInt(FS_INPUT_BUF_SIZE_ATTR, 256 * 1024);
}
+ @Private
+ @VisibleForTesting
+ public long getRollOverLogMaxSize(Configuration conf) {
+ return 1024L * 1024 * 1024 * conf.getInt(
+ LOG_ROLL_OVER_MAX_FILE_SIZE_GB, 10);
+ }
+
private abstract class FSAction<T> {
abstract T run() throws Exception;
@@ -1054,4 +1130,77 @@ public class LogAggregationIndexedFileController
}
}
}
+
+ private Path getCurrentRemoteLogFile(final FileContext fc,
+ final Path parent, final String nodeId) throws IOException {
+ RemoteIterator<FileStatus> files = fc.listStatus(parent);
+ long maxTime = 0L;
+ Path returnPath = null;
+ while(files.hasNext()) {
+ FileStatus candidate = files.next();
+ String fileName = candidate.getPath().getName();
+ if (fileName.contains(LogAggregationUtils.getNodeString(nodeId))
+ && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX) &&
+ !fileName.endsWith(CHECK_SUM_FILE_SUFFIX)) {
+ if (candidate.getModificationTime() > maxTime) {
+ maxTime = candidate.getModificationTime();
+ returnPath = candidate.getPath();
+ }
+ }
+ }
+ return returnPath;
+ }
+
+ private byte[] loadUUIDFromLogFile(final FileContext fc,
+ final Path parent, final ApplicationId appId, final String nodeId)
+ throws Exception {
+ byte[] id = null;
+ RemoteIterator<FileStatus> files = fc.listStatus(parent);
+ FSDataInputStream fsDataInputStream = null;
+ byte[] uuid = createUUID(appId);
+ while(files.hasNext()) {
+ try {
+ Path checkPath = files.next().getPath();
+ if (checkPath.getName().contains(LogAggregationUtils
+ .getNodeString(nodeId)) && !checkPath.getName()
+ .endsWith(CHECK_SUM_FILE_SUFFIX)) {
+ fsDataInputStream = fc.open(checkPath);
+ byte[] b = new byte[uuid.length];
+ int actual = fsDataInputStream.read(b);
+ if (actual != uuid.length || Arrays.equals(b, uuid)) {
+ deleteFileWithRetries(fc, checkPath);
+ } else if (id == null){
+ id = uuid;
+ }
+ }
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, fsDataInputStream);
+ }
+ }
+ return id == null ? uuid : id;
+ }
+
+ @Private
+ @VisibleForTesting
+ public boolean isRollover(final FileContext fc,
+ final Path candidate) throws IOException {
+ FileStatus fs = fc.getFileStatus(candidate);
+ return fs.getLen() >= this.logRollOverMaxFileSize;
+ }
+
+ @Private
+ @VisibleForTesting
+ public Clock getSystemClock() {
+ return SystemClock.getInstance();
+ }
+
+ private byte[] createUUID(ApplicationId appId) throws IOException {
+ try {
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ return digest.digest(appId.toString().getBytes(
+ Charset.forName("UTF-8")));
+ } catch (NoSuchAlgorithmException ex) {
+ throw new IOException(ex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/280080fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
index f77ad96..7d0205b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
@@ -21,13 +21,13 @@ package org.apache.hadoop.yarn.logaggregation.filecontroller.ifile;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Writer;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,6 +37,8 @@ import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -54,6 +56,8 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ControlledClock;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -143,8 +147,27 @@ public class TestLogAggregationIndexFileController {
LogValue value = mock(LogValue.class);
when(value.getPendingLogFilesToUploadForThisContainer()).thenReturn(files);
+ final ControlledClock clock = new ControlledClock();
+ clock.setTime(System.currentTimeMillis());
LogAggregationIndexedFileController fileFormat
- = new LogAggregationIndexedFileController();
+ = new LogAggregationIndexedFileController() {
+ private int rollOverCheck = 0;
+ @Override
+ public Clock getSystemClock() {
+ return clock;
+ }
+
+ @Override
+ public boolean isRollover(final FileContext fc,
+ final Path candidate) throws IOException {
+ rollOverCheck++;
+ if (rollOverCheck >= 3) {
+ return true;
+ }
+ return false;
+ }
+ };
+
fileFormat.initialize(conf, "Indexed");
Map<ApplicationAccessType, String> appAcls = new HashMap<>();
@@ -203,7 +226,11 @@ public class TestLogAggregationIndexFileController {
+ LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX);
FSDataOutputStream fInput = null;
try {
+ String nodeName = logPath.getName() + "_" + clock.getTime();
fInput = FileSystem.create(fs, checksumFile, LOG_FILE_UMASK);
+ fInput.writeInt(nodeName.length());
+ fInput.write(nodeName.getBytes(
+ Charset.forName("UTF-8")));
fInput.writeLong(0);
} finally {
IOUtils.closeQuietly(fInput);
@@ -236,9 +263,9 @@ public class TestLogAggregationIndexFileController {
// We did not call postWriter which we would keep the checksum file.
// We can only get the logs/logmeta from the first write.
- fileFormat.readAggregatedLogsMeta(
+ meta = fileFormat.readAggregatedLogsMeta(
logRequest);
- Assert.assertEquals(meta.size(), meta.size(), 1);
+ Assert.assertEquals(meta.size(), 1);
for (ContainerLogMeta log : meta) {
Assert.assertTrue(log.getContainerId().equals(containerId.toString()));
Assert.assertTrue(log.getNodeId().equals(nodeId.toString()));
@@ -267,9 +294,37 @@ public class TestLogAggregationIndexFileController {
fileFormat.write(key1, value2);
fileFormat.postWrite(context);
fileFormat.closeWriter();
- fileFormat.readAggregatedLogsMeta(
+ meta = fileFormat.readAggregatedLogsMeta(
logRequest);
- Assert.assertEquals(meta.size(), meta.size(), 2);
+ Assert.assertEquals(meta.size(), 2);
+ for (ContainerLogMeta log : meta) {
+ Assert.assertTrue(log.getContainerId().equals(containerId.toString()));
+ Assert.assertTrue(log.getNodeId().equals(nodeId.toString()));
+ for (ContainerLogFileInfo file : log.getContainerLogMeta()) {
+ fileNames.add(file.getFileName());
+ }
+ }
+ fileNames.removeAll(newLogTypes);
+ Assert.assertTrue(fileNames.isEmpty());
+ foundLogs = fileFormat.readAggregatedLogs(logRequest, System.out);
+ Assert.assertTrue(foundLogs);
+ for (String logType : newLogTypes) {
+ Assert.assertTrue(sysOutStream.toString().contains(logMessage(
+ containerId, logType)));
+ }
+ sysOutStream.reset();
+
+ // start to roll over old logs
+ clock.setTime(System.currentTimeMillis());
+ fileFormat.initializeWriter(context);
+ fileFormat.write(key1, value2);
+ fileFormat.postWrite(context);
+ fileFormat.closeWriter();
+ FileStatus[] status = fs.listStatus(logPath.getParent());
+ Assert.assertTrue(status.length == 2);
+ meta = fileFormat.readAggregatedLogsMeta(
+ logRequest);
+ Assert.assertEquals(meta.size(), 3);
for (ContainerLogMeta log : meta) {
Assert.assertTrue(log.getContainerId().equals(containerId.toString()));
Assert.assertTrue(log.getNodeId().equals(nodeId.toString()));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-6509. Add a size threshold
beyond which yarn logs will require a force option. (Xuan Gong via wangda)
Posted by kk...@apache.org.
YARN-6509. Add a size threshold beyond which yarn logs will require a force option. (Xuan Gong via wangda)
Change-Id: I755fe903337d4ff9ec35dae5b9cce638794e1d0f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec2ae306
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec2ae306
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec2ae306
Branch: refs/heads/YARN-6592
Commit: ec2ae3060a807c8754826af2135a68c08b2e4f13
Parents: febeead
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Sep 29 16:42:56 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Sep 29 16:42:56 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 366 ++++++++++++-------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 41 ++-
.../yarn/logaggregation/LogCLIHelpers.java | 8 +-
.../tfile/LogAggregationTFileController.java | 6 +
4 files changed, 279 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 9a8ba4a..74497ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -29,9 +29,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
@@ -107,6 +110,7 @@ public class LogsCLI extends Configured implements Tool {
private static final String CLIENT_RETRY_INTERVAL_OPTION
= "client_retry_interval_ms";
public static final String HELP_CMD = "help";
+ private static final String SIZE_LIMIT_OPTION = "size_limit_mb";
private PrintStream outStream = System.out;
private YarnClient yarnClient = null;
@@ -115,6 +119,11 @@ public class LogsCLI extends Configured implements Tool {
private static final int DEFAULT_MAX_RETRIES = 30;
private static final long DEFAULT_RETRY_INTERVAL = 1000;
+ private static final long LOG_SIZE_LIMIT_DEFAULT = 10240L;
+
+ private long logSizeLeft = LOG_SIZE_LIMIT_DEFAULT * 1024 * 1024;
+ private long specifedLogLimits = LOG_SIZE_LIMIT_DEFAULT;
+
@Private
@VisibleForTesting
ClientConnectionRetry connectionRetry;
@@ -158,6 +167,7 @@ public class LogsCLI extends Configured implements Tool {
List<String> amContainersList = new ArrayList<String>();
String localDir = null;
long bytes = Long.MAX_VALUE;
+ boolean ignoreSizeLimit = false;
int maxRetries = DEFAULT_MAX_RETRIES;
long retryInterval = DEFAULT_RETRY_INTERVAL;
try {
@@ -199,6 +209,14 @@ public class LogsCLI extends Configured implements Tool {
retryInterval = Long.parseLong(commandLine.getOptionValue(
CLIENT_RETRY_INTERVAL_OPTION));
}
+ if (commandLine.hasOption(SIZE_LIMIT_OPTION)) {
+ specifedLogLimits = Long.parseLong(commandLine.getOptionValue(
+ SIZE_LIMIT_OPTION));
+ logSizeLeft = specifedLogLimits * 1024 * 1024;
+ }
+ if (logSizeLeft < 0L) {
+ ignoreSizeLimit = true;
+ }
} catch (ParseException e) {
System.err.println("options parsing failed: " + e.getMessage());
printHelpMessage(printOpts);
@@ -306,6 +324,7 @@ public class LogsCLI extends Configured implements Tool {
logs.addAll(Arrays.asList(logFilesRegex));
}
+
ContainerLogsRequest request = new ContainerLogsRequest(appId,
isApplicationFinished(appState), appOwner, nodeAddress, null,
containerIdStr, localDir, logs, bytes, null);
@@ -324,15 +343,17 @@ public class LogsCLI extends Configured implements Tool {
// To get am logs
if (getAMContainerLogs) {
return fetchAMContainerLogs(request, amContainersList,
- logCliHelper, useRegex);
+ logCliHelper, useRegex, ignoreSizeLimit);
}
int resultCode = 0;
if (containerIdStr != null) {
- return fetchContainerLogs(request, logCliHelper, useRegex);
+ return fetchContainerLogs(request, logCliHelper, useRegex,
+ ignoreSizeLimit);
} else {
if (nodeAddress == null) {
- resultCode = fetchApplicationLogs(request, logCliHelper, useRegex);
+ resultCode = fetchApplicationLogs(request, logCliHelper, useRegex,
+ ignoreSizeLimit);
} else {
System.err.println("Should at least provide ContainerId!");
printHelpMessage(printOpts);
@@ -524,35 +545,16 @@ public class LogsCLI extends Configured implements Tool {
@VisibleForTesting
public int printContainerLogsFromRunningApplication(Configuration conf,
ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
+ boolean useRegex, boolean ignoreSizeLimit) throws IOException {
String containerIdStr = request.getContainerId().toString();
String localDir = request.getOutputLocalDir();
- String nodeHttpAddress = request.getNodeHttpAddress();
- if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
- System.err.println("Can not get the logs for the container: "
- + containerIdStr);
- System.err.println("The node http address is required to get container "
- + "logs for the Running application.");
- return -1;
- }
String nodeId = request.getNodeId();
PrintStream out = LogToolUtils.createPrintStream(localDir, nodeId,
containerIdStr);
try {
- Set<String> matchedFiles = getMatchedContainerLogFiles(request,
- useRegex);
- if (matchedFiles.isEmpty()) {
- System.err.println("Can not find any log file matching the pattern: "
- + request.getLogTypes() + " for the container: " + containerIdStr
- + " within the application: " + request.getAppId());
- return -1;
- }
- ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
- newOptions.setLogTypes(matchedFiles);
-
boolean foundAnyLogs = false;
byte[] buffer = new byte[65536];
- for (String logFile : newOptions.getLogTypes()) {
+ for (String logFile : request.getLogTypes()) {
InputStream is = null;
try {
ClientResponse response = getResponeFromNMWebService(conf,
@@ -595,50 +597,6 @@ public class LogsCLI extends Configured implements Tool {
}
}
- private int printContainerLogsForFinishedApplication(
- ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
- ContainerLogsRequest newOptions = getMatchedLogOptions(
- request, logCliHelper, useRegex);
- if (newOptions == null) {
- System.err.println("Can not find any log file matching the pattern: "
- + request.getLogTypes() + " for the container: "
- + request.getContainerId() + " within the application: "
- + request.getAppId());
- return -1;
- }
- return logCliHelper.dumpAContainerLogsForLogType(newOptions);
- }
-
- private int printContainerLogsForFinishedApplicationWithoutNodeId(
- ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
- ContainerLogsRequest newOptions = getMatchedLogOptions(
- request, logCliHelper, useRegex);
- if (newOptions == null) {
- System.err.println("Can not find any log file matching the pattern: "
- + request.getLogTypes() + " for the container: "
- + request.getContainerId() + " within the application: "
- + request.getAppId());
- return -1;
- }
- return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
- newOptions);
- }
-
- private int printAggregatedContainerLogs(ContainerLogsRequest request,
- LogCLIHelpers logCliHelper, boolean useRegex) throws IOException {
- return printContainerLogsForFinishedApplication(request,
- logCliHelper, useRegex);
- }
-
- private int printAggregatedContainerLogsWithoutNodeId(
- ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
- return printContainerLogsForFinishedApplicationWithoutNodeId(request,
- logCliHelper, useRegex);
- }
-
@Private
@VisibleForTesting
public ContainerReport getContainerReport(String containerIdStr)
@@ -655,7 +613,8 @@ public class LogsCLI extends Configured implements Tool {
private int printAMContainerLogs(Configuration conf,
ContainerLogsRequest request, List<String> amContainers,
- LogCLIHelpers logCliHelper, boolean useRegex) throws Exception {
+ LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
+ throws Exception {
List<JSONObject> amContainersList = null;
List<ContainerLogsRequest> requests =
new ArrayList<ContainerLogsRequest>();
@@ -717,10 +676,9 @@ public class LogsCLI extends Configured implements Tool {
return -1;
}
+ List<ContainerLogsRequest> candidates = new ArrayList<>();
if (amContainers.contains("ALL")) {
- for (ContainerLogsRequest amRequest : requests) {
- outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex);
- }
+ candidates.addAll(requests);
outStream.println();
outStream.println("Specified ALL for -am option. "
+ "Printed logs for all am containers.");
@@ -728,12 +686,10 @@ public class LogsCLI extends Configured implements Tool {
for (String amContainer : amContainers) {
int amContainerId = Integer.parseInt(amContainer.trim());
if (amContainerId == -1) {
- outputAMContainerLogs(requests.get(requests.size() - 1), conf,
- logCliHelper, useRegex);
+ candidates.add(requests.get(requests.size() - 1));
} else {
if (amContainerId <= requests.size()) {
- outputAMContainerLogs(requests.get(amContainerId - 1), conf,
- logCliHelper, useRegex);
+ candidates.add(requests.get(amContainerId - 1));
} else {
System.err.println(String.format("ERROR: Specified AM containerId"
+ " (%s) exceeds the number of AM containers (%s).",
@@ -743,12 +699,25 @@ public class LogsCLI extends Configured implements Tool {
}
}
}
+ Map<String, ContainerLogsRequest> newOptions = new HashMap<>();
+ if (request.isAppFinished()) {
+ newOptions = getMatchedLogTypesForFinishedApp(candidates,
+ logCliHelper, useRegex, ignoreSizeLimit);
+ } else {
+ newOptions = getMatchedLogTypesForRunningApp(candidates, useRegex,
+ ignoreSizeLimit);
+ }
+ for (Entry<String, ContainerLogsRequest> amRequest
+ : newOptions.entrySet()) {
+ outputAMContainerLogs(amRequest.getValue(), conf, logCliHelper,
+ useRegex, ignoreSizeLimit);
+ }
return 0;
}
private void outputAMContainerLogs(ContainerLogsRequest request,
- Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex)
- throws Exception {
+ Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex,
+ boolean ignoreSizeLimit) throws Exception {
String nodeHttpAddress = request.getNodeHttpAddress();
String containerId = request.getContainerId();
String nodeId = request.getNodeId();
@@ -756,11 +725,10 @@ public class LogsCLI extends Configured implements Tool {
if (request.isAppFinished()) {
if (containerId != null && !containerId.isEmpty()) {
if (nodeId != null && !nodeId.isEmpty()) {
- printContainerLogsForFinishedApplication(request,
- logCliHelper, useRegex);
+ logCliHelper.dumpAContainerLogsForLogType(request);
} else {
- printContainerLogsForFinishedApplicationWithoutNodeId(
- request, logCliHelper, useRegex);
+ logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
+ request);
}
}
} else {
@@ -770,7 +738,7 @@ public class LogsCLI extends Configured implements Tool {
.getContainerState();
request.setContainerState(containerState);
printContainerLogsFromRunningApplication(conf,
- request, logCliHelper, useRegex);
+ request, logCliHelper, useRegex, ignoreSizeLimit);
}
}
}
@@ -898,6 +866,13 @@ public class LogsCLI extends Configured implements Tool {
opts.addOption(CLIENT_RETRY_INTERVAL_OPTION, true,
"Work with --client_max_retries to create a retry client. "
+ "The default value is 1000.");
+ opts.addOption(SIZE_LIMIT_OPTION, true, "Use this option to limit "
+ + "the size of the total logs which could be fetched. "
+ + "By default, we only allow to fetch at most "
+ + LOG_SIZE_LIMIT_DEFAULT + " MB logs. If the total log size is "
+ + "larger than the specified number, the CLI would fail. "
+ + "The user could specify -1 to ignore the size limit "
+ + "and fetch all logs.");
opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID");
opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID");
opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address");
@@ -908,6 +883,7 @@ public class LogsCLI extends Configured implements Tool {
opts.getOption(CLIENT_MAX_RETRY_OPTION).setArgName("Max Retries");
opts.getOption(CLIENT_RETRY_INTERVAL_OPTION)
.setArgName("Retry Interval");
+ opts.getOption(SIZE_LIMIT_OPTION).setArgName("Size Limit");
return opts;
}
@@ -933,6 +909,7 @@ public class LogsCLI extends Configured implements Tool {
PER_CONTAINER_LOG_FILES_REGEX_OPTION));
printOpts.addOption(commandOpts.getOption(CLIENT_MAX_RETRY_OPTION));
printOpts.addOption(commandOpts.getOption(CLIENT_RETRY_INTERVAL_OPTION));
+ printOpts.addOption(commandOpts.getOption(SIZE_LIMIT_OPTION));
return printOpts;
}
@@ -969,15 +946,15 @@ public class LogsCLI extends Configured implements Tool {
private int fetchAMContainerLogs(ContainerLogsRequest request,
List<String> amContainersList, LogCLIHelpers logCliHelper,
- boolean useRegex) throws Exception {
+ boolean useRegex, boolean ignoreSizeLimit) throws Exception {
return printAMContainerLogs(getConf(), request, amContainersList,
- logCliHelper, useRegex);
+ logCliHelper, useRegex, ignoreSizeLimit);
}
private int fetchContainerLogs(ContainerLogsRequest request,
- LogCLIHelpers logCliHelper, boolean useRegex) throws IOException,
- ClientHandlerException, UniformInterfaceException, JSONException {
- int resultCode = 0;
+ LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
+ throws IOException, ClientHandlerException, UniformInterfaceException,
+ JSONException {
String appIdStr = request.getAppId().toString();
String containerIdStr = request.getContainerId();
String nodeAddress = request.getNodeId();
@@ -988,12 +965,20 @@ public class LogsCLI extends Configured implements Tool {
if (isAppFinished) {
// if user specified "ALL" as the logFiles param, pass empty list
// to logCliHelper so that it fetches all the logs
+ ContainerLogsRequest newOptions = getMatchedLogOptions(
+ request, logCliHelper, useRegex, ignoreSizeLimit);
+ if (newOptions == null) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + request.getLogTypes() + " for the container: "
+ + request.getContainerId() + " within the application: "
+ + request.getAppId());
+ return -1;
+ }
if (nodeAddress != null && !nodeAddress.isEmpty()) {
- return printContainerLogsForFinishedApplication(
- request, logCliHelper, useRegex);
+ return logCliHelper.dumpAContainerLogsForLogType(newOptions);
} else {
- return printContainerLogsForFinishedApplicationWithoutNodeId(
- request, logCliHelper, useRegex);
+ return logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
+ newOptions);
}
}
String nodeHttpAddress = null;
@@ -1019,13 +1004,20 @@ public class LogsCLI extends Configured implements Tool {
} else {
// for the case, we have already uploaded partial logs in HDFS
int result = -1;
- if (nodeAddress != null && !nodeAddress.isEmpty()) {
- result = printAggregatedContainerLogs(request,
- logCliHelper, useRegex);
+ ContainerLogsRequest newOptions = getMatchedLogOptions(
+ request, logCliHelper, useRegex, ignoreSizeLimit);
+ if (newOptions == null) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + request.getLogTypes() + " for the container: "
+ + request.getContainerId() + " within the application: "
+ + request.getAppId());
} else {
- result = printAggregatedContainerLogsWithoutNodeId(request,
- logCliHelper,
- useRegex);
+ if (nodeAddress != null && !nodeAddress.isEmpty()) {
+ result = logCliHelper.dumpAContainerLogsForLogType(newOptions);
+ } else {
+ result = logCliHelper.dumpAContainerLogsForLogTypeWithoutNodeId(
+ newOptions);
+ }
}
if (result == -1) {
System.err.println(
@@ -1043,14 +1035,18 @@ public class LogsCLI extends Configured implements Tool {
// If the application is not in the final state,
// we will provide the NodeHttpAddress and get the container logs
// by calling NodeManager webservice.
- resultCode = printContainerLogsFromRunningApplication(getConf(), request,
- logCliHelper, useRegex);
- return resultCode;
+ ContainerLogsRequest newRequest = getMatchedOptionForRunningApp(
+ request, useRegex, ignoreSizeLimit);
+ if (newRequest == null) {
+ return -1;
+ }
+ return printContainerLogsFromRunningApplication(getConf(), request,
+ logCliHelper, useRegex, ignoreSizeLimit);
}
private int fetchApplicationLogs(ContainerLogsRequest options,
- LogCLIHelpers logCliHelper, boolean useRegex) throws IOException,
- YarnException {
+ LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
+ throws IOException, YarnException {
// If the application has finished, we would fetch the logs
// from HDFS.
// If the application is still running, we would get the full
@@ -1059,7 +1055,7 @@ public class LogsCLI extends Configured implements Tool {
int resultCode = -1;
if (options.isAppFinished()) {
ContainerLogsRequest newOptions = getMatchedLogOptions(
- options, logCliHelper, useRegex);
+ options, logCliHelper, useRegex, ignoreSizeLimit);
if (newOptions == null) {
System.err.println("Can not find any log file matching the pattern: "
+ options.getLogTypes() + " for the application: "
@@ -1071,9 +1067,17 @@ public class LogsCLI extends Configured implements Tool {
} else {
List<ContainerLogsRequest> containerLogRequests =
getContainersLogRequestForRunningApplication(options);
- for (ContainerLogsRequest container : containerLogRequests) {
+
+ // get all matched container log types and check the total log size.
+ Map<String, ContainerLogsRequest> matchedLogTypes =
+ getMatchedLogTypesForRunningApp(containerLogRequests,
+ useRegex, ignoreSizeLimit);
+
+ for (Entry<String, ContainerLogsRequest> container
+ : matchedLogTypes.entrySet()) {
int result = printContainerLogsFromRunningApplication(getConf(),
- container, logCliHelper, useRegex);
+ container.getValue(), logCliHelper,
+ useRegex, ignoreSizeLimit);
if (result == 0) {
resultCode = 0;
}
@@ -1103,37 +1107,54 @@ public class LogsCLI extends Configured implements Tool {
private ContainerLogsRequest getMatchedLogOptions(
ContainerLogsRequest request, LogCLIHelpers logCliHelper,
- boolean useRegex) throws IOException {
+ boolean useRegex, boolean ignoreSizeLimit) throws IOException {
ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
- if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) {
- Set<String> matchedFiles = new HashSet<String>();
- if (!request.getLogTypes().contains("ALL")) {
- Set<String> files = logCliHelper.listContainerLogs(request);
- matchedFiles = getMatchedLogFiles(request, files, useRegex);
- if (matchedFiles.isEmpty()) {
- return null;
- }
- }
+ Set<ContainerLogFileInfo> files = logCliHelper.listContainerLogs(
+ request);
+ Set<String> matchedFiles = getMatchedLogFiles(request, files,
+ useRegex, ignoreSizeLimit);
+ if (matchedFiles.isEmpty()) {
+ return null;
+ } else {
newOptions.setLogTypes(matchedFiles);
+ return newOptions;
}
- return newOptions;
}
private Set<String> getMatchedLogFiles(ContainerLogsRequest options,
- Collection<String> candidate, boolean useRegex) throws IOException {
+ Collection<ContainerLogFileInfo> candidate, boolean useRegex,
+ boolean ignoreSizeLimit) throws IOException {
Set<String> matchedFiles = new HashSet<String>();
Set<String> filePattern = options.getLogTypes();
- if (options.getLogTypes().contains("ALL")) {
- return new HashSet<String>(candidate);
- }
- for (String file : candidate) {
- if (useRegex) {
- if (isFileMatching(file, filePattern)) {
- matchedFiles.add(file);
+ long size = options.getBytes();
+ boolean getAll = options.getLogTypes().contains("ALL");
+ Iterator<ContainerLogFileInfo> iterator = candidate.iterator();
+ while(iterator.hasNext()) {
+ boolean matchedFile = false;
+ ContainerLogFileInfo logInfo = iterator.next();
+ if (getAll) {
+ matchedFile = true;
+ } else if (useRegex) {
+ if (isFileMatching(logInfo.getFileName(), filePattern)) {
+ matchedFile = true;
}
} else {
- if (filePattern.contains(file)) {
- matchedFiles.add(file);
+ if (filePattern.contains(logInfo.getFileName())) {
+ matchedFile = true;
+ }
+ }
+ if (matchedFile) {
+ matchedFiles.add(logInfo.getFileName());
+ if (!ignoreSizeLimit) {
+ decrLogSizeLimit(Math.min(
+ Long.parseLong(logInfo.getFileSize()), size));
+ if (getLogSizeLimitLeft() < 0) {
+ throw new RuntimeException("The total log size is too large."
+ + "The log size limit is " + specifedLogLimits + "MB. "
+ + "Please specify a proper value --size option or if you "
+ + "really want to fetch all, please "
+ + "specify -1 for --size_limit_mb option.");
+ }
}
}
}
@@ -1296,18 +1317,19 @@ public class LogsCLI extends Configured implements Tool {
@VisibleForTesting
public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request,
- boolean useRegex) throws IOException {
+ boolean useRegex, boolean ignoreSizeLimit) throws IOException {
// fetch all the log files for the container
// filter the log files based on the given -log_files pattern
List<Pair<ContainerLogFileInfo, String>> allLogFileInfos=
getContainerLogFiles(getConf(), request.getContainerId(),
request.getNodeHttpAddress());
- List<String> fileNames = new ArrayList<String>();
+ List<ContainerLogFileInfo> fileNames = new ArrayList<
+ ContainerLogFileInfo>();
for (Pair<ContainerLogFileInfo, String> fileInfo : allLogFileInfos) {
- fileNames.add(fileInfo.getKey().getFileName());
+ fileNames.add(fileInfo.getKey());
}
return getMatchedLogFiles(request, fileNames,
- useRegex);
+ useRegex, ignoreSizeLimit);
}
@VisibleForTesting
@@ -1451,4 +1473,86 @@ public class LogsCLI extends Configured implements Tool {
// The method to indicate if we should retry given the incoming exception
public abstract boolean shouldRetryOn(Exception e);
}
+
+ private long getLogSizeLimitLeft() {
+ return this.logSizeLeft;
+ }
+
+ private void decrLogSizeLimit(long used) {
+ this.logSizeLeft -= used;
+ }
+
+ @Private
+ @VisibleForTesting
+ public ContainerLogsRequest getMatchedOptionForRunningApp(
+ ContainerLogsRequest container, boolean useRegex,
+ boolean ignoreSizeLimit) throws IOException {
+ String containerIdStr = container.getContainerId().toString();
+ String nodeHttpAddress = container.getNodeHttpAddress();
+ if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
+ System.err.println("Can not get the logs for the container: "
+ + containerIdStr);
+ System.err.println("The node http address is required to get container "
+ + "logs for the Running application.");
+ return null;
+ }
+
+ Set<String> matchedFiles = getMatchedContainerLogFiles(container,
+ useRegex, ignoreSizeLimit);
+ if (matchedFiles.isEmpty()) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + container.getLogTypes() + " for the container: " + containerIdStr
+ + " within the application: " + container.getAppId());
+ return null;
+ }
+ container.setLogTypes(matchedFiles);
+ return container;
+ }
+
+ @Private
+ @VisibleForTesting
+ public Map<String, ContainerLogsRequest> getMatchedLogTypesForRunningApp(
+ List<ContainerLogsRequest> containerLogRequests, boolean useRegex,
+ boolean ignoreSizeLimit) {
+ Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>();
+ for (ContainerLogsRequest container : containerLogRequests) {
+ try {
+ ContainerLogsRequest request = getMatchedOptionForRunningApp(
+ container, useRegex, ignoreSizeLimit);
+ if (request == null) {
+ continue;
+ }
+ containerMatchedLog.put(container.getContainerId(), request);
+ } catch(IOException ex) {
+ System.err.println(ex);
+ continue;
+ }
+ }
+ return containerMatchedLog;
+ }
+
+ private Map<String, ContainerLogsRequest> getMatchedLogTypesForFinishedApp(
+ List<ContainerLogsRequest> containerLogRequests,
+ LogCLIHelpers logCliHelper, boolean useRegex,
+ boolean ignoreSizeLimit) {
+ Map<String, ContainerLogsRequest> containerMatchedLog = new HashMap<>();
+ for (ContainerLogsRequest container : containerLogRequests) {
+ try {
+ ContainerLogsRequest request = getMatchedLogOptions(container,
+ logCliHelper, useRegex, ignoreSizeLimit);
+ if (request == null) {
+ System.err.println("Can not find any log file matching the pattern: "
+ + container.getLogTypes() + " for the container: "
+ + container.getContainerId() + " within the application: "
+ + container.getAppId());
+ continue;
+ }
+ containerMatchedLog.put(container.getContainerId(), request);
+ } catch (IOException ex) {
+ System.err.println(ex);
+ continue;
+ }
+ }
+ return containerMatchedLog;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index fed7488..c292430 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -30,6 +30,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Matchers.*;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
@@ -300,6 +301,17 @@ public class TestLogsCLI {
pw.println(" end and positive values as");
pw.println(" bytes to read from the");
pw.println(" beginning.");
+ pw.println(" -size_limit_mb <Size Limit> Use this option to limit the");
+ pw.println(" size of the total logs which");
+ pw.println(" could be fetched. By");
+ pw.println(" default, we only allow to");
+ pw.println(" fetch at most 10240 MB logs.");
+ pw.println(" If the total log size is");
+ pw.println(" larger than the specified");
+ pw.println(" number, the CLI would fail.");
+ pw.println(" The user could specify -1 to");
+ pw.println(" ignore the size limit and");
+ pw.println(" fetch all logs.");
pw.close();
String appReportStr = baos.toString("UTF-8");
Assert.assertTrue(sysOutStream.toString().contains(appReportStr));
@@ -563,8 +575,7 @@ public class TestLogsCLI {
containerId0.toString() });
assertTrue(exitCode == -1);
assertTrue(sysErrStream.toString().contains(
- "Logs for container " + containerId0.toString()
- + " are not present in this log-file."));
+ "Can not find any log file matching the pattern"));
sysErrStream.reset();
// uploaded two logs for container3. The first log is named as syslog.
@@ -750,7 +761,7 @@ public class TestLogsCLI {
Set<String> logsSet = new HashSet<String>();
logsSet.add(fileName);
doReturn(logsSet).when(cli).getMatchedContainerLogFiles(
- any(ContainerLogsRequest.class), anyBoolean());
+ any(ContainerLogsRequest.class), anyBoolean(), anyBoolean());
ClientResponse mockReponse = mock(ClientResponse.class);
doReturn(Status.OK).when(mockReponse).getStatusInfo();
doReturn(fis).when(mockReponse).getEntityInputStream();
@@ -795,6 +806,7 @@ public class TestLogsCLI {
doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
doReturn("http://localhost:2345").when(mockContainerReport1)
.getNodeHttpAddress();
+
ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
ContainerReport mockContainerReport2 = mock(ContainerReport.class);
doReturn(containerId2).when(mockContainerReport2).getContainerId();
@@ -812,7 +824,19 @@ public class TestLogsCLI {
LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
doReturn(0).when(cli).printContainerLogsFromRunningApplication(
any(Configuration.class), any(ContainerLogsRequest.class),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
+ Set<String> logTypes = new HashSet<>();
+ logTypes.add("ALL");
+ ContainerLogsRequest mockContainer1 = mock(ContainerLogsRequest.class);
+ doReturn(logTypes).when(mockContainer1).getLogTypes();
+ ContainerLogsRequest mockContainer2 = mock(ContainerLogsRequest.class);
+ doReturn(logTypes).when(mockContainer2).getLogTypes();
+ Map<String, ContainerLogsRequest> matchedLogTypes = new HashMap<>();
+ matchedLogTypes.put(containerId1.toString(), mockContainer1);
+ matchedLogTypes.put(containerId2.toString(), mockContainer2);
+ doReturn(matchedLogTypes).when(cli).getMatchedLogTypesForRunningApp(
+ anyListOf(ContainerLogsRequest.class), anyBoolean(),
+ anyBoolean());
cli.setConf(new YarnConfiguration());
int exitCode = cli.run(new String[] {"-applicationId", appId.toString()});
@@ -825,7 +849,7 @@ public class TestLogsCLI {
// printContainerLogsFromRunningApplication twice
verify(cli, times(2)).printContainerLogsFromRunningApplication(
any(Configuration.class), logsRequestCaptor.capture(),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
// Verify that the log-type is "ALL"
List<ContainerLogsRequest> capturedRequests =
@@ -839,9 +863,12 @@ public class TestLogsCLI {
mockYarnClient = createMockYarnClientWithException(
YarnApplicationState.RUNNING, ugi.getShortUserName());
LogsCLI cli2 = spy(new LogsCLIForTest(mockYarnClient));
+ ContainerLogsRequest newOption = mock(ContainerLogsRequest.class);
+ doReturn(newOption).when(cli2).getMatchedOptionForRunningApp(
+ any(ContainerLogsRequest.class), anyBoolean(), anyBoolean());
doReturn(0).when(cli2).printContainerLogsFromRunningApplication(
any(Configuration.class), any(ContainerLogsRequest.class),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
doReturn("123").when(cli2).getNodeHttpAddressFromRMWebString(
any(ContainerLogsRequest.class));
cli2.setConf(new YarnConfiguration());
@@ -851,7 +878,7 @@ public class TestLogsCLI {
assertTrue(exitCode == 0);
verify(cli2, times(1)).printContainerLogsFromRunningApplication(
any(Configuration.class), logsRequestCaptor.capture(),
- any(LogCLIHelpers.class), anyBoolean());
+ any(LogCLIHelpers.class), anyBoolean(), anyBoolean());
}
@Test (timeout = 15000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 97b78ec..887d92d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -350,10 +350,10 @@ public class LogCLIHelpers implements Configurable {
}
@Private
- public Set<String> listContainerLogs(ContainerLogsRequest options)
- throws IOException {
+ public Set<ContainerLogFileInfo> listContainerLogs(
+ ContainerLogsRequest options) throws IOException {
List<ContainerLogMeta> containersLogMeta;
- Set<String> logTypes = new HashSet<String>();
+ Set<ContainerLogFileInfo> logTypes = new HashSet<ContainerLogFileInfo>();
try {
containersLogMeta = getFileController(options.getAppId(),
options.getAppOwner()).readAggregatedLogsMeta(
@@ -364,7 +364,7 @@ public class LogCLIHelpers implements Configurable {
}
for (ContainerLogMeta logMeta: containersLogMeta) {
for (ContainerLogFileInfo fileInfo : logMeta.getContainerLogMeta()) {
- logTypes.add(fileInfo.getFileName());
+ logTypes.add(fileInfo);
}
}
return logTypes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec2ae306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
index 989b326..92e3a08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java
@@ -268,6 +268,12 @@ public class LogAggregationTFileController
}
while (nodeFiles.hasNext()) {
FileStatus thisNodeFile = nodeFiles.next();
+ if (thisNodeFile.getPath().getName().equals(appId + ".har")) {
+ Path p = new Path("har:///"
+ + thisNodeFile.getPath().toUri().getRawPath());
+ nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p);
+ continue;
+ }
if (nodeIdStr != null) {
if (!thisNodeFile.getPath().getName().contains(nodeIdStr)) {
continue;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDFS-12458.
TestReencryptionWithKMS fails regularly. Contributed by Xiao Chen.
Posted by kk...@apache.org.
HDFS-12458. TestReencryptionWithKMS fails regularly. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c34ceaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c34ceaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c34ceaf
Branch: refs/heads/YARN-6592
Commit: 7c34ceaf4fa28e2ecabd6626860bb1c7418e4b8d
Parents: 28c4957
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Sep 28 05:09:37 2017 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Sep 28 05:10:26 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/TestReencryption.java | 13 +++++++------
.../hdfs/server/namenode/TestReencryptionHandler.java | 7 ++++---
2 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c34ceaf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
index 33c52bf..4bf6aa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -129,11 +128,9 @@ public class TestReencryption {
conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 3);
// Adjust configs for re-encrypt test cases
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REENCRYPT_BATCH_SIZE_KEY, 5);
- conf.setTimeDuration(
- DFSConfigKeys.DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY, 1,
- TimeUnit.SECONDS);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
+ cluster.waitClusterUp();
fs = cluster.getFileSystem();
fsn = cluster.getNamesystem();
fsWrapper = new FileSystemTestWrapper(fs);
@@ -1284,6 +1281,7 @@ public class TestReencryption {
fsn = cluster.getNamesystem();
getEzManager().pauseReencryptForTesting();
cluster.waitActive();
+ cluster.waitClusterUp();
}
private void waitForReencryptedZones(final int expected)
@@ -1519,6 +1517,7 @@ public class TestReencryption {
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForQueuedZones(1);
+ getEzManager().pauseReencryptUpdaterForTesting();
getEzManager().resumeReencryptForTesting();
LOG.info("Waiting for re-encrypt callables to run");
@@ -1529,7 +1528,6 @@ public class TestReencryption {
}
}, 100, 10000);
- getEzManager().pauseReencryptUpdaterForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.CANCEL);
// now resume updater and verify status.
@@ -1609,7 +1607,7 @@ public class TestReencryption {
cluster.getConfiguration(0)
.unset(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
cluster.restartNameNodes();
- cluster.waitActive();
+ cluster.waitClusterUp();
// test re-encrypt should fail
try {
@@ -1673,6 +1671,9 @@ public class TestReencryption {
}
fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ // trigger the background thread to run, without having to
+ // wait for DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY
+ getHandler().notifyNewSubmission();
waitForReencryptedFiles(zone.toString(), 10);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c34ceaf/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
index f0dd92c..e2035ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
@@ -101,9 +101,9 @@ public class TestReencryptionHandler {
final StopWatch sw = new StopWatch().start();
rh.throttle();
sw.stop();
- assertTrue("should have throttled for at least 4 second",
+ assertTrue("should have throttled for at least 8 second",
sw.now(TimeUnit.MILLISECONDS) > 8000);
- assertTrue("should have throttled for at most 6 second",
+ assertTrue("should have throttled for at most 12 second",
sw.now(TimeUnit.MILLISECONDS) < 12000);
}
@@ -191,7 +191,8 @@ public class TestReencryptionHandler {
removeTaskThread.start();
rh.throttle();
sw.stop();
+ LOG.info("Throttle completed, consumed {}", sw.now(TimeUnit.MILLISECONDS));
assertTrue("should have throttled for at least 3 second",
- sw.now(TimeUnit.MILLISECONDS) > 3000);
+ sw.now(TimeUnit.MILLISECONDS) >= 3000);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-14908. CrossOriginFilter
should trigger regex on more input (Johannes Alberti via aw)
Posted by kk...@apache.org.
HADOOP-14908. CrossOriginFilter should trigger regex on more input (Johannes Alberti via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d5dd75b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d5dd75b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d5dd75b
Branch: refs/heads/YARN-6592
Commit: 4d5dd75b607d25adf8b41f7408713dfcea8f5330
Parents: 4111e6c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Oct 3 10:58:28 2017 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Oct 3 10:58:28 2017 -0700
----------------------------------------------------------------------
.../hadoop/security/http/CrossOriginFilter.java | 27 +++++--
.../src/main/resources/core-default.xml | 12 ++-
.../src/site/markdown/HttpAuthentication.md | 2 +-
.../security/http/TestCrossOriginFilter.java | 79 ++++++++++++++++++++
.../src/site/markdown/TimelineServer.md | 32 ++++----
5 files changed, 125 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
index 58d50cf..34d9fe2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
@@ -37,6 +37,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import com.google.common.annotations.VisibleForTesting;
+import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,6 +67,7 @@ public class CrossOriginFilter implements Filter {
// Filter configuration
public static final String ALLOWED_ORIGINS = "allowed-origins";
public static final String ALLOWED_ORIGINS_DEFAULT = "*";
+ public static final String ALLOWED_ORIGINS_REGEX_PREFIX = "regex:";
public static final String ALLOWED_METHODS = "allowed-methods";
public static final String ALLOWED_METHODS_DEFAULT = "GET,POST,HEAD";
public static final String ALLOWED_HEADERS = "allowed-headers";
@@ -194,6 +196,12 @@ public class CrossOriginFilter implements Filter {
allowAllOrigins = allowedOrigins.contains("*");
LOG.info("Allowed Origins: " + StringUtils.join(allowedOrigins, ','));
LOG.info("Allow All Origins: " + allowAllOrigins);
+ List<String> discouragedAllowedOrigins = allowedOrigins.stream()
+ .filter(s -> s.length() > 1 && s.contains("*"))
+ .collect(Collectors.toList());
+ for (String discouragedAllowedOrigin : discouragedAllowedOrigins) {
+ LOG.warn("Allowed Origin pattern '" + discouragedAllowedOrigin + "' is discouraged, use the 'regex:' prefix and use a Java regular expression instead.");
+ }
}
private void initializeMaxAge(FilterConfig filterConfig) {
@@ -228,15 +236,20 @@ public class CrossOriginFilter implements Filter {
String[] origins = originsList.trim().split("\\s+");
for (String origin : origins) {
for (String allowedOrigin : allowedOrigins) {
- if (allowedOrigin.contains("*")) {
- String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
- Pattern p = Pattern.compile(regex);
- Matcher m = p.matcher(origin);
- if (m.matches()) {
+ Pattern regexPattern = null;
+ if (allowedOrigin.startsWith(ALLOWED_ORIGINS_REGEX_PREFIX)) {
+ String regex = allowedOrigin.substring(ALLOWED_ORIGINS_REGEX_PREFIX.length());
+ regexPattern = Pattern.compile(regex);
+ } else if (allowedOrigin.contains("*")) {
+ String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
+ regexPattern = Pattern.compile(regex);
+ }
+
+ if (regexPattern != null
+ && regexPattern.matcher(origin).matches()) {
return true;
- }
} else if (allowedOrigin.equals(origin)) {
- return true;
+ return true;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index a11e7c3..d481048 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1861,9 +1861,15 @@
<property>
<name>hadoop.http.cross-origin.allowed-origins</name>
<value>*</value>
- <description>Comma separated list of origins that are allowed for web
- services needing cross-origin (CORS) support. Wildcards (*) and patterns
- allowed</description>
+ <description>Comma separated list of origins that are allowed for web services
+ needing cross-origin (CORS) support. If a value in the list contains an
+ asterix (*), a regex pattern, escaping any dots ('.' -> '\.') and replacing
+ the asterix such that it captures any characters ('*' -> '.*'), is generated.
+ Values prefixed with 'regex:' are interpreted directly as regular expressions,
+ e.g. use the expression 'regex:https?:\/\/foo\.bar:([0-9]+)?' to allow any
+ origin using the 'http' or 'https' protocol in the domain 'foo.bar' on any
+ port. The use of simple wildcards ('*') is discouraged, and only available for
+ backward compatibility.</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
index d4eeb0e..44d814c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
@@ -60,7 +60,7 @@ Add org.apache.hadoop.security.HttpCrossOriginFilterInitializer to hadoop.http.f
| Property | Default Value | Description |
|:---------------------------------------- |:--------------------------------------------- |:------------------------------------------------------------------------------------- |
| hadoop.http.cross-origin.enabled | `false` | Enables cross origin support for all web-services |
-| hadoop.http.cross-origin.allowed-origins | `*` | Comma separated list of origins that are allowed, wildcards (`*`) and patterns allowed |
+| hadoop.http.cross-origin.allowed-origins | `*` | Comma separated list of origins that are allowed. Values prefixed with `regex:` are interpreted as regular expressions. Values containing wildcards (`*`) are possible as well, here a regular expression is generated, the use is discouraged and support is only available for backward compatibility. |
| hadoop.http.cross-origin.allowed-methods | `GET,POST,HEAD` | Comma separated list of methods that are allowed |
| hadoop.http.cross-origin.allowed-headers | `X-Requested-With,Content-Type,Accept,Origin` | Comma separated list of headers that are allowed |
| hadoop.http.cross-origin.max-age | `1800` | Number of seconds a pre-flighted request can be cached |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
index 5c9b413..b9662b8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
@@ -128,6 +128,85 @@ public class TestCrossOriginFilter {
}
@Test
+ public void testRegexPatternMatchingOrigins() throws ServletException, IOException {
+
+ // Setup the configuration settings of the server
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "regex:.*[.]example[.]com");
+ FilterConfig filterConfig = new FilterConfigTest(conf);
+
+ // Object under test
+ CrossOriginFilter filter = new CrossOriginFilter();
+ filter.init(filterConfig);
+
+ // match multiple sub-domains
+ Assert.assertFalse(filter.areOriginsAllowed("example.com"));
+ Assert.assertFalse(filter.areOriginsAllowed("foo:example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.bar.example.com"));
+
+ // First origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example.com foo.nomatch.com"));
+ // Second origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.nomatch.com foo.example.com"));
+ // No origin in list is allowed
+ Assert.assertFalse(filter.areOriginsAllowed("foo.nomatch1.com foo.nomatch2.com"));
+ }
+
+ @Test
+ public void testComplexRegexPatternMatchingOrigins() throws ServletException, IOException {
+
+ // Setup the configuration settings of the server
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "regex:https?:\\/\\/sub1[.]example[.]com(:[0-9]+)?");
+ FilterConfig filterConfig = new FilterConfigTest(conf);
+
+ // Object under test
+ CrossOriginFilter filter = new CrossOriginFilter();
+ filter.init(filterConfig);
+
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com:1234"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com:8080"));
+
+ // No origin in list is allowed
+ Assert.assertFalse(filter.areOriginsAllowed("foo.nomatch1.com foo.nomatch2.com"));
+ }
+
+ @Test
+ public void testMixedRegexPatternMatchingOrigins() throws ServletException, IOException {
+
+ // Setup the configuration settings of the server
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "regex:https?:\\/\\/sub1[.]example[.]com(:[0-9]+)?, "
+ + "*.example2.com");
+ FilterConfig filterConfig = new FilterConfigTest(conf);
+
+ // Object under test
+ CrossOriginFilter filter = new CrossOriginFilter();
+ filter.init(filterConfig);
+
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com:1234"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com:8080"));
+
+ // match multiple sub-domains
+ Assert.assertFalse(filter.areOriginsAllowed("example2.com"));
+ Assert.assertFalse(filter.areOriginsAllowed("foo:example2.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example2.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.bar.example2.com"));
+
+ // First origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example2.com foo.nomatch.com"));
+ // Second origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.nomatch.com foo.example2.com"));
+ // No origin in list is allowed
+ Assert.assertFalse(filter.areOriginsAllowed("foo.nomatch1.com foo.nomatch2.com"));
+ }
+
+ @Test
public void testDisallowedOrigin() throws ServletException, IOException {
// Setup the configuration settings of the server
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index f610cde..a326f45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -41,9 +41,9 @@ Previously this was supported purely for MapReduce jobs by the Application Histo
With the introduction of the timeline server, the Application History Server becomes just one use of
the Timeline Server.
-Generic information includes application level data such as
+Generic information includes application level data such as
-* queue-name,
+* queue-name,
* user information and the like set in the `ApplicationSubmissionContext`,
* a list of application-attempts that ran for an application
* information about each application-attempt
@@ -68,7 +68,7 @@ Current status
1. The "Timeline Server v1" REST API has been declared one of the REST APIs
whose compatibility will be maintained in future releases.
1. The single-server implementation of the Timeline Server places a limit on
- the scalability of the service; it also prevents the service being
+ the scalability of the service; it also prevents the service being
High-Availability component of the YARN infrastructure.
Future Plans
@@ -88,9 +88,9 @@ data structures as well as the ability of the client to failover between Timelin
The Timeline Domain offers a namespace for Timeline server allowing
users to host multiple entities, isolating them from other users and applications.
Timeline server Security is defined at this level.
-
+
A "Domain" primarily stores owner info, read and& write ACL information,
-created and modified time stamp information. Each Domain is identified by an ID which
+created and modified time stamp information. Each Domain is identified by an ID which
must be unique across all users in the YARN cluster.
#### Timeline Entity
@@ -111,7 +111,7 @@ Each Entity is uniquely identified by an `EntityId` and `EntityType`.
#### Timeline Events
A Timeline Event describes an event that is related to a specific
-Timeline Entity of an application.
+Timeline Entity of an application.
Users are free to define what an event means —such as starting
an application, getting allocated a container,
@@ -156,7 +156,7 @@ and cluster operators.
| `yarn.timeline-service.webapp.https.address` | The https address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8190`. |
| `yarn.timeline-service.bind-host` | The actual address the server will bind to. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in `yarn.timeline-service.address` and `yarn.timeline-service.webapp.address`, respectively. This is most useful for making the service listen on all interfaces by setting to `0.0.0.0`. |
| `yarn.timeline-service.http-cross-origin.enabled` | Enables cross-origin support (CORS) for web services where cross-origin web response headers are needed. For example, javascript making a web services request to the timeline server. Defaults to `false`. |
-| `yarn.timeline-service.http-cross-origin.allowed-origins` | Comma separated list of origins that are allowed for web services needing cross-origin (CORS) support. Wildcards `(*)` and patterns allowed. Defaults to `*`. |
+| `yarn.timeline-service.http-cross-origin.allowed-origins` | Comma separated list of origins that are allowed. Values prefixed with `regex:` are interpreted as regular expressions. Values containing wildcards (`*`) are possible as well, here a regular expression is generated, the use is discouraged and support is only available for backward compatibility. Defaults to `*`. |
| `yarn.timeline-service.http-cross-origin.allowed-methods` | Comma separated list of methods that are allowed for web services needing cross-origin (CORS) support. Defaults to `GET,POST,HEAD`. |
| `yarn.timeline-service.http-cross-origin.allowed-headers` | Comma separated list of headers that are allowed for web services needing cross-origin (CORS) support. Defaults to `X-Requested-With,Content-Type,Accept,Origin`. |
| `yarn.timeline-service.http-cross-origin.max-age` | The number of seconds a pre-flighted request can be cached for web services needing cross-origin (CORS) support. Defaults to `1800`. |
@@ -420,7 +420,7 @@ response: `TimelinePutResponse`
### List domains of a user: GET `/ws/v1/timeline/domain`
-Retrieves a list of all domains of a user.
+Retrieves a list of all domains of a user.
If an owner is specified, that owner name overrides that of the caller.
@@ -598,8 +598,8 @@ Request Body:
Required fields
Entity: `type` and `id`. `starttime` is required unless the
-entity contains one or more event).
-Event: `type` and `timestamp`.
+entity contains one or more event).
+Event: `type` and `timestamp`.
## <a name="REST_API_LIST_TIMELINE_ENTITIES"></a>Timeline Entity List
@@ -809,7 +809,7 @@ Response Body:
}
]
}
-
+
@@ -1443,8 +1443,8 @@ None
### Elements of the `appattempts` (Application Attempt List) Object
When you make a request for the list of application attempts, the information
-will be returned as a collection of application attempt objects. See
-[Application Attempt](#REST_API_APPLICATION_ATTEMPT) for the syntax of
+will be returned as a collection of application attempt objects. See
+[Application Attempt](#REST_API_APPLICATION_ATTEMPT) for the syntax of
the application attempt object.
| Item | Data Type | Description |
@@ -1758,7 +1758,7 @@ Response Body:
}
]
}
-
+
#### XML response
@@ -2004,8 +2004,8 @@ Response Body:
<containerState>COMPLETE</containerState>
<nodeHttpAddress>http://localhost:8042</nodeHttpAddress>
</container>
-
-
+
+
### Response Codes
1. Queries where a domain, entity type, entity ID or similar cannot be resolved result in
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by kk...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
index 60fce40..05b44b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
@@ -114,8 +114,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("rm " + MOCK_CONTAINER_ID, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=rm", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
}
@Test
@@ -130,8 +132,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("stop " + MOCK_CONTAINER_ID, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=stop", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
}
@Test
@@ -147,9 +151,12 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("inspect --format='{{.State.Status}}' " + MOCK_CONTAINER_ID,
- dockerCommands.get(0));
+ assertEquals(4, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=inspect", dockerCommands.get(1));
+ assertEquals(" format={{.State.Status}}", dockerCommands.get(2));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(3));
+
}
@Test
@@ -165,8 +172,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("pull " + MOCK_IMAGE_NAME, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=pull", dockerCommands.get(1));
+ assertEquals(" image=" + MOCK_IMAGE_NAME, dockerCommands.get(2));
}
@Test
@@ -182,8 +191,12 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("load --i=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=load", dockerCommands.get(1));
+ assertEquals(" image=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(2));
+
+
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
index 619f202..4092e6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -44,16 +46,29 @@ public class TestDockerInspectCommand {
@Test
public void testGetContainerStatus() throws Exception {
dockerInspectCommand.getContainerStatus();
- assertEquals("inspect --format='{{.State.Status}}' foo",
- dockerInspectCommand.getCommandWithArguments());
+ assertEquals("inspect", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("{{.State.Status}}", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("format")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(3,
+ dockerInspectCommand.getDockerCommandWithArguments().size());
}
@Test
public void testGetIpAndHost() throws Exception {
dockerInspectCommand.getIpAndHost();
- assertEquals(
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}}"
- + ",{{end}}{{.Config.Hostname}}' foo",
- dockerInspectCommand.getCommandWithArguments());
+ assertEquals("inspect", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("format")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(3,
+ dockerInspectCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
index 85fa0f8..e5bff26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +43,11 @@ public class TestDockerLoadCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("load --i=foo",
- dockerLoadCommand.getCommandWithArguments());
+ assertEquals("load", StringUtils.join(",",
+ dockerLoadCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerLoadCommand.getDockerCommandWithArguments().get("image")));
+ assertEquals(2, dockerLoadCommand.getDockerCommandWithArguments().size());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
index 89157ff..ccf7000 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +43,12 @@ public class TestDockerPullCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("pull foo", dockerPullCommand.getCommandWithArguments());
+ assertEquals("pull", StringUtils.join(",",
+ dockerPullCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerPullCommand.getDockerCommandWithArguments().get("image")));
+ assertEquals(2, dockerPullCommand.getDockerCommandWithArguments().size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
index d1b9904..a8d4bdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +44,11 @@ public class TestDockerRmCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("rm foo", dockerRmCommand.getCommandWithArguments());
+ assertEquals("rm", StringUtils.join(",",
+ dockerRmCommand.getDockerCommandWithArguments().get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRmCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(2, dockerRmCommand.getDockerCommandWithArguments().size());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
index 85bccd2..e51d7ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -56,8 +57,24 @@ public class TestDockerRunCommand {
commands.add("launch_command");
dockerRunCommand.setOverrideCommandWithArgs(commands);
dockerRunCommand.removeContainerOnExit();
- assertEquals("run --name=foo --user=user_id --device=source:dest --rm "
- + "image_name launch_command",
- dockerRunCommand.getCommandWithArguments());
+
+ assertEquals("run", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals("user_id", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("user")));
+ assertEquals("image_name", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("image")));
+
+ assertEquals("source:dest", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("devices")));
+ assertEquals("true", StringUtils
+ .join(",", dockerRunCommand.getDockerCommandWithArguments().get("rm")));
+ assertEquals("launch_command", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments()
+ .get("launch-command")));
+ assertEquals(7, dockerRunCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
index c9743f3..efbde77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
@@ -21,6 +21,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -48,8 +50,13 @@ public class TestDockerStopCommand {
@Test
public void testSetGracePeriod() throws Exception {
dockerStopCommand.setGracePeriod(GRACE_PERIOD);
- assertEquals("stop foo --time=10",
- dockerStopCommand.getCommandWithArguments());
-
+ assertEquals("stop", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals("10", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments().get("time")));
+ assertEquals(3, dockerStopCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 23f4134..36c391a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -167,7 +167,24 @@ The following properties are required to enable Docker support:
|Configuration Name | Description |
|:---- |:---- |
| `yarn.nodemanager.linux-container-executor.group` | The Unix group of the NodeManager. It should match the yarn.nodemanager.linux-container-executor.group in the yarn-site.xml file. |
-| `feature.docker.enabled` | Must be 0 or 1. 0 means launching Docker containers is disabled. 1 means launching Docker containers is allowed. |
+
+The container-executor.cfg must contain a section to determine the capabilities that containers
+are allowed. It contains the following properties:
+
+|Configuration Name | Description |
+|:---- |:---- |
+| `module.enabled` | Must be "true" or "false" to enable or disable launching Docker containers respectively. Default value is 0. |
+| `docker.binary` | The binary used to launch Docker containers. /usr/bin/docker by default. |
+| `docker.allowed.capabilities` | Comma separated capabilities that containers are allowed to add. By default no capabilities are allowed to be added. |
+| `docker.allowed.devices` | Comma separated devices that containers are allowed to mount. By default no devices are allowed to be added. |
+| `docker.allowed.networks` | Comma separated networks that containers are allowed to use. If no network is specified when launching the container, the default Docker network will be used. |
+| `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to mount in read-only mode. By default, no directories are allowed to mounted. |
+| `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. |
+| `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged containers. Default value is 0. |
+
+Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list.
+
+In addition, containers are not permitted to mount any parent of the container-executor.cfg directory in read-write mode.
The following properties are optional:
@@ -176,9 +193,21 @@ The following properties are optional:
| `min.user.id` | The minimum UID that is allowed to launch applications. The default is no minimum |
| `banned.users` | A comma-separated list of usernames who should not be allowed to launch applications. The default setting is: yarn, mapred, hdfs, and bin. |
| `allowed.system.users` | A comma-separated list of usernames who should be allowed to launch applications even if their UIDs are below the configured minimum. If a user appears in allowed.system.users and banned.users, the user will be considered banned. |
-| `docker.binary` | The path to the Docker binary. The default is "docker". |
| `feature.tc.enabled` | Must be 0 or 1. 0 means traffic control commands are disabled. 1 means traffic control commands are allowed. |
+Part of a container-executor.cfg which allows Docker containers to be launched is below:
+
+```
+yarn.nodemanager.linux-container-executor.group=yarn
+[docker]
+ module.enabled=true
+ docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE
+ docker.allowed.networks=bridge,host,none
+ docker.allowed.ro-mounts=/sys/fs/cgroup
+ docker.allowed.rw-mounts=/var/hadoop/yarn/local-dir,/var/hadoop/yarn/log-dir
+
+```
+
Docker Image Requirements
-------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-14822.
hadoop-project/pom.xml is executable. Contributed by Ajay Kumar.
Posted by kk...@apache.org.
HADOOP-14822. hadoop-project/pom.xml is executable. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd378775
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd378775
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd378775
Branch: refs/heads/YARN-6592
Commit: dd378775df5bd79e23c12120535404c4ca0fb386
Parents: 0da29cb
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Sep 27 14:25:07 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Sep 27 14:25:07 2017 +0900
----------------------------------------------------------------------
hadoop-project/pom.xml | 0
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd378775/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
old mode 100755
new mode 100644
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-7253. Shared Cache Manager
daemon command listed as admin subcmd in yarn script.
Posted by kk...@apache.org.
YARN-7253. Shared Cache Manager daemon command listed as admin subcmd in yarn script.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c87db8d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c87db8d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c87db8d1
Branch: refs/heads/YARN-6592
Commit: c87db8d154ab2501e786b4f1669b205759ece5c3
Parents: 8facf1f
Author: Chris Trezzo <ct...@apache.org>
Authored: Wed Sep 27 11:32:09 2017 -0700
Committer: Chris Trezzo <ct...@apache.org>
Committed: Wed Sep 27 11:32:09 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c87db8d1/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index dcde0dc..52138c5 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -48,7 +48,7 @@ function hadoop_usage
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon"
+ hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
hadoop_add_subcommand "top" client "view cluster information"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: Revert "YARN-6570. No logs were
found for running application, running"
Posted by kk...@apache.org.
Revert "YARN-6570. No logs were found for running application, running"
This reverts commit 7c732924a889cd280e972882619a1827877fbafa.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14fec04e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14fec04e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14fec04e
Branch: refs/heads/YARN-6592
Commit: 14fec04ed335ae5f442abd3e35ec5fe05be62360
Parents: e9b790d
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 26 10:38:03 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 26 10:38:03 2017 -0500
----------------------------------------------------------------------
.../nodemanager/containermanager/container/ContainerImpl.java | 1 -
.../org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java | 3 +--
.../nodemanager/containermanager/container/TestContainer.java | 3 ---
3 files changed, 1 insertion(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14fec04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 705087b..2bbc267 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -709,7 +709,6 @@ public class ContainerImpl implements Container {
public org.apache.hadoop.yarn.api.records.ContainerState getCurrentState() {
switch (stateMachine.getCurrentState()) {
case NEW:
- return org.apache.hadoop.yarn.api.records.ContainerState.NEW;
case LOCALIZING:
case LOCALIZATION_FAILED:
case SCHEDULED:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14fec04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
index 9e59449..8e4522b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
@@ -159,8 +159,7 @@ public class TestEventFlow {
containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
- Arrays.asList(ContainerState.RUNNING, ContainerState.SCHEDULED,
- ContainerState.NEW), 20);
+ Arrays.asList(ContainerState.RUNNING, ContainerState.SCHEDULED), 20);
List<ContainerId> containerIds = new ArrayList<ContainerId>();
containerIds.add(cID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14fec04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index b44b500..64e6cf0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -122,8 +122,6 @@ public class TestContainer {
try {
wc = new WrappedContainer(7, 314159265358979L, 4344, "yak");
assertEquals(ContainerState.NEW, wc.c.getContainerState());
- ContainerImpl container = (ContainerImpl)wc.c;
- assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.NEW, container.getCurrentState());
wc.initContainer();
// Verify request for public/private resources to localizer
@@ -133,7 +131,6 @@ public class TestContainer {
LocalResourceVisibility.APPLICATION));
verify(wc.localizerBus).handle(argThat(matchesReq));
assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState());
- assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED, container.getCurrentState());
}
finally {
if (wc != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-6962. Add support for
updateContainers when allocating using FederationInterceptor. (Botong Huang
via Subru).
Posted by kk...@apache.org.
YARN-6962. Add support for updateContainers when allocating using FederationInterceptor. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca669f9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca669f9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca669f9f
Branch: refs/heads/YARN-6592
Commit: ca669f9f8bc7abe5b7d4648c589aa1756bd336d1
Parents: 85d81ae5
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Sep 28 13:04:03 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Thu Sep 28 13:04:03 2017 -0700
----------------------------------------------------------------------
.../amrmproxy/FederationInterceptor.java | 86 +++++++++++++-------
.../amrmproxy/TestFederationInterceptor.java | 54 ++++++++++++
2 files changed, 111 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca669f9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index 28724aa..33cfca3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -540,30 +540,33 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
}
}
- if (request.getResourceBlacklistRequest() != null && !isNullOrEmpty(
- request.getResourceBlacklistRequest().getBlacklistAdditions())) {
- for (String resourceName : request.getResourceBlacklistRequest()
- .getBlacklistAdditions()) {
- SubClusterId subClusterId = getSubClusterForNode(resourceName);
- if (subClusterId != null) {
- AllocateRequest newRequest = findOrCreateAllocateRequestForSubCluster(
- subClusterId, request, requestMap);
- newRequest.getResourceBlacklistRequest().getBlacklistAdditions()
- .add(resourceName);
+ if (request.getResourceBlacklistRequest() != null) {
+ if (!isNullOrEmpty(
+ request.getResourceBlacklistRequest().getBlacklistAdditions())) {
+ for (String resourceName : request.getResourceBlacklistRequest()
+ .getBlacklistAdditions()) {
+ SubClusterId subClusterId = getSubClusterForNode(resourceName);
+ if (subClusterId != null) {
+ AllocateRequest newRequest =
+ findOrCreateAllocateRequestForSubCluster(subClusterId, request,
+ requestMap);
+ newRequest.getResourceBlacklistRequest().getBlacklistAdditions()
+ .add(resourceName);
+ }
}
}
- }
-
- if (request.getResourceBlacklistRequest() != null && !isNullOrEmpty(
- request.getResourceBlacklistRequest().getBlacklistRemovals())) {
- for (String resourceName : request.getResourceBlacklistRequest()
- .getBlacklistRemovals()) {
- SubClusterId subClusterId = getSubClusterForNode(resourceName);
- if (subClusterId != null) {
- AllocateRequest newRequest = findOrCreateAllocateRequestForSubCluster(
- subClusterId, request, requestMap);
- newRequest.getResourceBlacklistRequest().getBlacklistRemovals()
- .add(resourceName);
+ if (!isNullOrEmpty(
+ request.getResourceBlacklistRequest().getBlacklistRemovals())) {
+ for (String resourceName : request.getResourceBlacklistRequest()
+ .getBlacklistRemovals()) {
+ SubClusterId subClusterId = getSubClusterForNode(resourceName);
+ if (subClusterId != null) {
+ AllocateRequest newRequest =
+ findOrCreateAllocateRequestForSubCluster(subClusterId, request,
+ requestMap);
+ newRequest.getResourceBlacklistRequest().getBlacklistRemovals()
+ .add(resourceName);
+ }
}
}
}
@@ -896,13 +899,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
}
}
- if (!isNullOrEmpty(otherResponse.getNMTokens())) {
- if (!isNullOrEmpty(homeResponse.getNMTokens())) {
- homeResponse.getNMTokens().addAll(otherResponse.getNMTokens());
- } else {
- homeResponse.setNMTokens(otherResponse.getNMTokens());
- }
- }
+ homeResponse.setNumClusterNodes(
+ homeResponse.getNumClusterNodes() + otherResponse.getNumClusterNodes());
PreemptionMessage homePreempMessage = homeResponse.getPreemptionMessage();
PreemptionMessage otherPreempMessage = otherResponse.getPreemptionMessage();
@@ -935,6 +933,31 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
spar1.getContainers().addAll(spar2.getContainers());
}
}
+
+ if (!isNullOrEmpty(otherResponse.getNMTokens())) {
+ if (!isNullOrEmpty(homeResponse.getNMTokens())) {
+ homeResponse.getNMTokens().addAll(otherResponse.getNMTokens());
+ } else {
+ homeResponse.setNMTokens(otherResponse.getNMTokens());
+ }
+ }
+
+ if (!isNullOrEmpty(otherResponse.getUpdatedContainers())) {
+ if (!isNullOrEmpty(homeResponse.getUpdatedContainers())) {
+ homeResponse.getUpdatedContainers()
+ .addAll(otherResponse.getUpdatedContainers());
+ } else {
+ homeResponse.setUpdatedContainers(otherResponse.getUpdatedContainers());
+ }
+ }
+
+ if (!isNullOrEmpty(otherResponse.getUpdateErrors())) {
+ if (!isNullOrEmpty(homeResponse.getUpdateErrors())) {
+ homeResponse.getUpdateErrors().addAll(otherResponse.getUpdateErrors());
+ } else {
+ homeResponse.setUpdateErrors(otherResponse.getUpdateErrors());
+ }
+ }
}
/**
@@ -1052,6 +1075,11 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
return this.uamPool.getAllUAMIds().size();
}
+ @VisibleForTesting
+ public Map<SubClusterId, List<AllocateResponse>> getAsyncResponseSink() {
+ return this.asyncResponseSink;
+ }
+
/**
* Private structure for encapsulating SubClusterId and
* RegisterApplicationMasterResponse instances.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca669f9f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
index 34b0741..3db0e35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
@@ -36,8 +38,15 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.PreemptionMessage;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerError;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -493,4 +502,49 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
} catch (YarnException e) {
}
}
+
+ @Test
+ public void testAllocateResponse() throws Exception {
+ interceptor.registerApplicationMaster(
+ RegisterApplicationMasterRequest.newInstance(null, 0, null));
+ AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
+
+ Map<SubClusterId, List<AllocateResponse>> asyncResponseSink =
+ interceptor.getAsyncResponseSink();
+
+ ContainerId cid = ContainerId.newContainerId(attemptId, 0);
+ ContainerStatus cStatus = Records.newRecord(ContainerStatus.class);
+ cStatus.setContainerId(cid);
+ Container container =
+ Container.newInstance(cid, null, null, null, null, null);
+
+ AllocateResponse response = Records.newRecord(AllocateResponse.class);
+ response.setAllocatedContainers(Collections.singletonList(container));
+ response.setCompletedContainersStatuses(Collections.singletonList(cStatus));
+ response.setUpdatedNodes(
+ Collections.singletonList(Records.newRecord(NodeReport.class)));
+ response.setNMTokens(
+ Collections.singletonList(Records.newRecord(NMToken.class)));
+ response.setUpdatedContainers(
+ Collections.singletonList(Records.newRecord(UpdatedContainer.class)));
+ response.setUpdateErrors(Collections
+ .singletonList(Records.newRecord(UpdateContainerError.class)));
+ response.setAvailableResources(Records.newRecord(Resource.class));
+ response.setPreemptionMessage(Records.newRecord(PreemptionMessage.class));
+
+ List<AllocateResponse> list = new ArrayList<>();
+ list.add(response);
+ asyncResponseSink.put(SubClusterId.newInstance("SC-1"), list);
+
+ response = interceptor.allocate(allocateRequest);
+
+ Assert.assertEquals(1, response.getAllocatedContainers().size());
+ Assert.assertNotNull(response.getAvailableResources());
+ Assert.assertEquals(1, response.getCompletedContainersStatuses().size());
+ Assert.assertEquals(1, response.getUpdatedNodes().size());
+ Assert.assertNotNull(response.getPreemptionMessage());
+ Assert.assertEquals(1, response.getNMTokens().size());
+ Assert.assertEquals(1, response.getUpdatedContainers().size());
+ Assert.assertEquals(1, response.getUpdateErrors().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDFS-12560. Remove the extra word
it in HdfsUserGuide.md. Contributed by fang zhenyi.
Posted by kk...@apache.org.
HDFS-12560. Remove the extra word it in HdfsUserGuide.md. Contributed by fang zhenyi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e1bd114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e1bd114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e1bd114
Branch: refs/heads/YARN-6592
Commit: 8e1bd114ee718c42cd3f3275bee20f5d401a1d99
Parents: 7c34cea
Author: Chen Liang <cl...@apache.org>
Authored: Thu Sep 28 10:22:27 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Thu Sep 28 10:22:27 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e1bd114/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
index 7a06c41..6f707f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
@@ -257,7 +257,7 @@ During start up the NameNode loads the file system state from the fsimage and th
fsck
----
-HDFS supports the fsck command to check for various inconsistencies. It it is designed for reporting problems with various files, for example, missing blocks for a file or under-replicated blocks. Unlike a traditional fsck utility for native file systems, this command does not correct the errors it detects. Normally NameNode automatically corrects most of the recoverable failures. By default fsck ignores open files but provides an option to select all files during reporting. The HDFS fsck command is not a Hadoop shell command. It can be run as `bin/hdfs fsck`. For command usage, see [fsck](./HDFSCommands.html#fsck). fsck can be run on the whole file system or on a subset of files.
+HDFS supports the fsck command to check for various inconsistencies. It is designed for reporting problems with various files, for example, missing blocks for a file or under-replicated blocks. Unlike a traditional fsck utility for native file systems, this command does not correct the errors it detects. Normally NameNode automatically corrects most of the recoverable failures. By default fsck ignores open files but provides an option to select all files during reporting. The HDFS fsck command is not a Hadoop shell command. It can be run as `bin/hdfs fsck`. For command usage, see [fsck](./HDFSCommands.html#fsck). fsck can be run on the whole file system or on a subset of files.
fetchdt
-------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-14890 Move up to AWS SDK
1.11.199. Contributed by Steve Loughran.
Posted by kk...@apache.org.
HADOOP-14890 Move up to AWS SDK 1.11.199. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2b31e35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2b31e35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2b31e35
Branch: refs/heads/YARN-6592
Commit: a2b31e355a73c37dada15a18a90a690314be7fd3
Parents: df800f6
Author: Aaron Fabbri <fa...@apache.org>
Authored: Mon Sep 25 17:11:12 2017 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Mon Sep 25 17:12:01 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b31e35/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7b342c4..915bd4e 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -135,7 +135,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.11.134</aws-java-sdk.version>
+ <aws-java-sdk.version>1.11.199</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<!-- the version of Hadoop declared in the version resources; can be overridden
so that Hadoop 3.x can declare itself a 2.x artifact. -->
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-6871. Add additional
deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
Posted by kk...@apache.org.
YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8facf1f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8facf1f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8facf1f9
Branch: refs/heads/YARN-6592
Commit: 8facf1f976d7e12a846f12baabf54be1b7a49f9d
Parents: dd37877
Author: Sunil G <su...@apache.org>
Authored: Wed Sep 27 14:37:32 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Sep 27 14:37:32 2017 +0530
----------------------------------------------------------------------
.../resourcemanager/webapp/DeSelectFields.java | 11 +-
.../resourcemanager/webapp/dao/AppInfo.java | 108 +++++++++++++------
.../webapp/TestRMWebServicesApps.java | 77 ++++++++++++-
3 files changed, 160 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
index 258bbfa..c991766 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
@@ -91,7 +91,16 @@ public class DeSelectFields {
* <code>RESOURCE_REQUESTS</code> is the first
* supported type from YARN-6280.
*/
- RESOURCE_REQUESTS("resourceRequests");
+ RESOURCE_REQUESTS("resourceRequests"),
+ /**
+ * <code>APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION,
+ * RESOURCE_INFO</code> are additionally supported parameters added in
+ * YARN-6871.
+ */
+ TIMEOUTS("timeouts"),
+ APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"),
+ AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"),
+ RESOURCE_INFO("resourceInfo");
private final String literals;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 236c467..40d7b4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -123,7 +123,7 @@ public class AppInfo {
protected String amNodeLabelExpression;
protected ResourcesInfo resourceInfo = null;
- protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
+ private AppTimeoutsInfo timeouts;
public AppInfo() {
} // JAXB needs this
@@ -247,47 +247,87 @@ public class AppInfo {
unmanagedApplication = appSubmissionContext.getUnmanagedAM();
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
- amNodeLabelExpression = (unmanagedApplication) ? null
- : app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ /*
+ * When the deSelects parameter contains "amNodeLabelExpression", objects
+ * pertaining to the amNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+ amNodeLabelExpression = (unmanagedApplication) ?
+ null :
+ app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "appNodeLabelExpression", objects
+ * pertaining to the appNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) {
+ appNodeLabelExpression =
+ app.getApplicationSubmissionContext().getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "amNodeLabelExpression", objects
+ * pertaining to the amNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+ amNodeLabelExpression = (unmanagedApplication) ?
+ null :
+ app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "resourceInfo", ResourceInfo
+ * objects are not returned. Default behavior is no skipping. (YARN-6871)
+ */
// Setting partition based resource usage of application
- ResourceScheduler scheduler = rm.getRMContext().getScheduler();
- if (scheduler instanceof CapacityScheduler) {
- RMAppAttempt attempt = app.getCurrentAppAttempt();
- if (null != attempt) {
- FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
- .getApplicationAttempt(attempt.getAppAttemptId());
- resourceInfo = null != ficaAppAttempt
- ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage())
- : null;
+ if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ if (scheduler instanceof CapacityScheduler) {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ if (null != attempt) {
+ FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attempt.getAppAttemptId());
+ resourceInfo = null != ficaAppAttempt ?
+ new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) :
+ null;
+ }
}
}
- Map<ApplicationTimeoutType, Long> applicationTimeouts =
- app.getApplicationTimeouts();
- if (applicationTimeouts.isEmpty()) {
- // If application is not set timeout, lifetime should be sent as default
- // with expiryTime=UNLIMITED and remainingTime=-1
- AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
- timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
- timeouts.add(timeoutInfo);
- } else {
- for (Map.Entry<ApplicationTimeoutType, Long> entry : app
- .getApplicationTimeouts().entrySet()) {
- AppTimeoutInfo timeout = new AppTimeoutInfo();
- timeout.setTimeoutType(entry.getKey());
- long timeoutInMillis = entry.getValue().longValue();
- timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
- if (app.isAppInCompletedStates()) {
- timeout.setRemainingTime(0);
- } else {
- timeout.setRemainingTime(Math
- .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ /*
+ * When the deSelects parameter contains "appTimeouts", objects pertaining
+ * to app timeouts are not returned. By default, this is not skipped.
+ * (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.TIMEOUTS)) {
+ Map<ApplicationTimeoutType, Long> applicationTimeouts =
+ app.getApplicationTimeouts();
+ if (applicationTimeouts.isEmpty()) {
+ // If application is not set timeout, lifetime should be sent
+ // as default with expiryTime=UNLIMITED and remainingTime=-1
+ AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+ timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+ timeouts = new AppTimeoutsInfo();
+ timeouts.add(timeoutInfo);
+ } else {
+ for (Map.Entry<ApplicationTimeoutType, Long> entry : app
+ .getApplicationTimeouts().entrySet()) {
+ AppTimeoutInfo timeout = new AppTimeoutInfo();
+ timeout.setTimeoutType(entry.getKey());
+ long timeoutInMillis = entry.getValue().longValue();
+ timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
+ if (app.isAppInCompletedStates()) {
+ timeout.setRemainingTime(0);
+ } else {
+ timeout.setRemainingTime(Math.max(
+ (timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ }
+ timeouts.add(timeout);
}
- timeouts.add(timeout);
}
}
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 07f74a3..f0704ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1113,8 +1113,83 @@ public class TestRMWebServicesApps extends JerseyTestBase {
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
JSONObject app = array.getJSONObject(0);
- assertTrue("resource requests shouldn't exits",
+ assertTrue("resource requests shouldn't exist",
!app.has("resourceRequests"));
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AMNodeLabelExpression shouldn't exist",
+ !app.has("amNodeLabelExpression"));
+
+ params.clear();
+ params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Timeouts shouldn't exist", !app.has("timeouts"));
+ rm.stop();
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AppNodeLabelExpression shouldn't exist",
+ !app.has("appNodeLabelExpression"));
+ rm.stop();
+
+ params.clear();
+ params
+ .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Resource info shouldn't exist", !app.has("resourceInfo"));
rm.stop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by kk...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
new file mode 100644
index 0000000..c627ca8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -0,0 +1,1122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <fstream>
+#include "errno.h"
+
+extern "C" {
+#include "utils/docker-util.c"
+}
+
+namespace ContainerExecutor {
+
+ class TestDockerUtil : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ docker_command_file = "docker-command.cmd";
+ container_executor_cfg_file = "container-executor.cfg";
+ container_executor_cfg.size = 0;
+ container_executor_cfg.sections = NULL;
+ }
+
+ virtual void TearDown() {
+ remove(docker_command_file.c_str());
+ remove(container_executor_cfg_file.c_str());
+ delete_ce_file();
+ }
+
+ struct configuration container_executor_cfg;
+ std::string docker_command_file;
+ std::string container_executor_cfg_file;
+
+
+ void write_file(const std::string fname, const std::string contents) {
+ std::ofstream command_file;
+ command_file.open(fname.c_str());
+ command_file << contents;
+ command_file.close();
+ }
+
+ int create_ce_file() {
+ int ret = 0;
+ const char *fname = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ if (strcmp("../etc/hadoop/container-executor.cfg", fname) == 0) {
+ ret = mkdir("../etc", 0755);
+ if (ret == 0 || errno == EEXIST) {
+ ret = mkdir("../etc/hadoop", 0755);
+ if (ret == 0 || errno == EEXIST) {
+ write_file("../etc/hadoop/container-executor.cfg", "");
+ return 0;
+ } else {
+ std::cerr << "Could not create ../etc/hadoop, " << strerror(errno) << std::endl;
+ }
+ } else {
+ std::cerr << "Could not create ../etc, " << strerror(errno) << std::endl;
+ }
+ }
+ std::cerr << "Could not create " << fname << std::endl;
+ return 1;
+ }
+
+ void delete_ce_file() {
+ const char *fname = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ if (strcmp("../etc/hadoop/container-executor.cfg", fname) == 0) {
+ struct stat buffer;
+ if (stat(fname, &buffer) == 0) {
+ remove("../etc/hadoop/container-executor.cfg");
+ rmdir("../etc/hadoop");
+ rmdir("../etc");
+ }
+ }
+ }
+
+ void write_container_executor_cfg(const std::string contents) {
+ write_file(container_executor_cfg_file, contents);
+ }
+
+ void write_command_file(const std::string contents) {
+ write_file(docker_command_file, contents);
+ }
+
+ void run_docker_command_test(const std::vector<std::pair<std::string, std::string> > &file_cmd_vec,
+ const std::vector<std::pair<std::string, int> > &bad_file_cmd_vec,
+ int (*docker_func)(const char *, const struct configuration *, char *, const size_t)) {
+ char tmp[8192];
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(tmp, 0, 8192);
+ write_command_file(itr->first);
+ int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(0, ret) << "error message: " << get_docker_error_message(ret) << " for input " << itr->first;
+ ASSERT_STREQ(itr->second.c_str(), tmp);
+ }
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+ for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
+ memset(tmp, 0, 8192);
+ write_command_file(itr2->first);
+ int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(itr2->second, ret) << " for " << itr2->first << std::endl;
+ ASSERT_EQ(0, strlen(tmp));
+ }
+ int ret = (*docker_func)("unknown-file", &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(static_cast<int>(INVALID_COMMAND_FILE), ret);
+ }
+
+ void run_docker_run_helper_function(const std::vector<std::pair<std::string, std::string> > &file_cmd_vec,
+ int (*helper_func)(const struct configuration *, char *, const size_t)) {
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for(itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ struct configuration cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ int ret = read_config(docker_command_file.c_str(), &cfg);
+ if(ret == 0) {
+ ret = (*helper_func)(&cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ }
+ }
+ };
+
+ TEST_F(TestDockerUtil, test_docker_inspect) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}\n name=container_e1_12312_11111_02_000001",
+ "inspect --format={{.State.Status}} container_e1_12312_11111_02_000001"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n"
+ " format={{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}\n"
+ " name=container_e1_12312_11111_02_000001",
+ "inspect --format={{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}} container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n format='{{.State.Status}}'",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(
+ std::make_pair<std::string, int>("docker-command=inspect\n format='{{.State.Status}}'",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}\n name=",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format=\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.IPAddress}}\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_inspect_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_load) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=load\n image=image-id",
+ "load --i='image-id' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=load\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=load\n image=", static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>("[docker-command-execution]\n docker-command=load",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_load_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_validate_image_name) {
+ const char *good_input[] = {
+ "ubuntu",
+ "ubuntu:latest",
+ "ubuntu:14.04",
+ "ubuntu:LATEST",
+ "registry.com:5000/user/ubuntu",
+ "registry.com:5000/user/ubuntu:latest",
+ "registry.com:5000/user/ubuntu:0.1.2.3",
+ "registry.com/user/ubuntu",
+ "registry.com/user/ubuntu:latest",
+ "registry.com/user/ubuntu:0.1.2.3",
+ "registry.com/user/ubuntu:test-image",
+ "registry.com/user/ubuntu:test_image",
+ "registry.com/ubuntu",
+ "user/ubuntu",
+ "user/ubuntu:0.1.2.3",
+ "user/ubuntu:latest",
+ "user/ubuntu:test_image",
+ "user/ubuntu.test:test_image",
+ "user/ubuntu-test:test-image",
+ "registry.com/ubuntu/ubuntu/ubuntu"
+ };
+
+ const char *bad_input[] = {
+ "UBUNTU",
+ "registry.com|5000/user/ubuntu",
+ "registry.com | 5000/user/ubuntu",
+ "ubuntu' || touch /tmp/file #",
+ "ubuntu || touch /tmp/file #",
+ "''''''''",
+ "bad_host_name:5000/user/ubuntu",
+ "registry.com:foo/ubuntu/ubuntu/ubuntu",
+ "registry.com/ubuntu:foo/ubuntu/ubuntu"
+ };
+
+ int good_input_size = sizeof(good_input) / sizeof(char *);
+ int i = 0;
+ for (i = 0; i < good_input_size; i++) {
+ int op = validate_docker_image_name(good_input[i]);
+ ASSERT_EQ(0, op);
+ }
+
+ int bad_input_size = sizeof(bad_input) / sizeof(char *);
+ int j = 0;
+ for (j = 0; j < bad_input_size; j++) {
+ int op = validate_docker_image_name(bad_input[j]);
+ ASSERT_EQ(1, op);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_pull) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=pull\n image=image-id",
+ "pull 'image-id' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=pull\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=pull\n image=", static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>("[docker-command-execution]\n docker-command=pull",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_pull_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_rm) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(
+ std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=rm\n name=container_e1_12312_11111_02_000001",
+ "rm container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=rm\n name=ctr-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=rm\n name=", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=rm", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_rm_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_stop) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001",
+ "stop container_e1_12312_11111_02_000001"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001\ntime=25",
+ "stop --time=25 container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=stop\n name=ctr-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop\n name=", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001\n time=abcd",
+ static_cast<int>(INVALID_DOCKER_STOP_COMMAND)));
+
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_stop_command);
+ }
+
+ TEST_F(TestDockerUtil, test_detach_container) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n detach=true", "-d "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, detach_container);
+ }
+
+ TEST_F(TestDockerUtil, test_rm_container) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rm=true", "--rm "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, rm_container_on_exit);
+ }
+
+ TEST_F(TestDockerUtil, test_set_container_workdir) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n workdir=/tmp/test", "--workdir='/tmp/test' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_container_workdir);
+ }
+
+ TEST_F(TestDockerUtil, test_set_cgroup_parent) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cgroup-parent=/sys/fs/cgroup/yarn",
+ "--cgroup-parent='/sys/fs/cgroup/yarn' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_cgroup_parent);
+ }
+
+ TEST_F(TestDockerUtil, test_set_hostname) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n hostname=ctr-id", "--hostname='ctr-id' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_hostname);
+ }
+
+ TEST_F(TestDockerUtil, test_set_group_add) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n group-add=1000,1001", "--group-add '1000' --group-add '1001' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_group_add);
+ }
+
+ TEST_F(TestDockerUtil, test_set_network) {
+ struct configuration container_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.networks=sdn1,bridge";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n net=bridge", "--net='bridge' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n net=sdn1", "--net='sdn1' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ struct configuration cmd_cfg;
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_network(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ struct configuration cmd_cfg_1;
+ write_command_file("[docker-command-execution]\n docker-command=run\n net=sdn2");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg_1);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_network(&cmd_cfg_1, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_network(&cmd_cfg_1, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_check_mount_permitted) {
+ const char *permitted_mounts[] = {"/usr/", "/bin/ls", NULL};
+ std::vector<std::pair<std::string, int> > test_data;
+ test_data.push_back(std::make_pair<std::string, int>("/usr", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/usr/", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/bin/ls", 1));
+ test_data.push_back(std::make_pair<std::string, int>("//bin/", 0));
+ test_data.push_back(std::make_pair<std::string, int>("/tmp/random-file", -1));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr;
+ for (itr = test_data.begin(); itr != test_data.end(); ++itr) {
+ int ret = check_mount_permitted(permitted_mounts, itr->first.c_str());
+ ASSERT_EQ(itr->second, ret) << "for input " << itr->first;
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_normalize_mounts) {
+ const int entries = 4;
+ const char *permitted_mounts[] = {"/home", "/usr", "/bin/ls", NULL};
+ const char *expected[] = {"/home/", "/usr/", "/bin/ls", NULL};
+ char **ptr = static_cast<char **>(malloc(entries * sizeof(char *)));
+ for (int i = 0; i < entries; ++i) {
+ if (permitted_mounts[i] != NULL) {
+ ptr[i] = strdup(permitted_mounts[i]);
+ } else {
+ ptr[i] = NULL;
+ }
+ }
+ normalize_mounts(ptr);
+ for (int i = 0; i < entries; ++i) {
+ ASSERT_STREQ(expected[i], ptr[i]);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_set_privileged) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1",
+ "[docker]\n docker.privileged-containers.enabled=0",
+ "[docker]\n"};
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=true", "--privileged "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=false", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ write_container_executor_cfg(container_executor_cfg_contents[0]);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ // check default case and when it's turned off
+ for (int i = 1; i < 3; ++i) {
+ write_container_executor_cfg(container_executor_cfg_contents[i]);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ file_cmd_vec.clear();
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=false", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n privileged=true");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_set_capabilities) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.capabilities=CHROOT,MKNOD";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT,MKNOD",
+ "--cap-drop='ALL' --cap-add='CHROOT' --cap-add='MKNOD' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT", "--cap-drop='ALL' --cap-add='CHROOT' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", "--cap-drop='ALL' "));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n cap-add=SETGID");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_set_devices) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.devices=/dev/test-device,/dev/device2";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n devices=/dev/test-device:/dev/test-device",
+ "--device='/dev/test-device:/dev/test-device' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n devices=/dev/device2:/dev/device2",
+ "--device='/dev/device2:/dev/device2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n "
+ "devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2",
+ "--device='/dev/test-device:/dev/test-device' --device='/dev/device2:/dev/device2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device3:/dev/device3");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device1");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+
+ TEST_F(TestDockerUtil, test_add_rw_mounts) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls,..\n "
+ "docker.allowed.ro-mounts=/bin/cat";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/var:/var", "-v '/var:/var' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/var/:/var/", "-v '/var/:/var/' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr:/usr", "-v '/usr:/usr' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/:/usr", "-v '/usr/:/usr' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin:/mydisk1,/var/log/:/mydisk2",
+ "-v '/usr/bin:/mydisk1' -v '/var/log/:/mydisk2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/home:/home",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/cat:/bin/cat",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/blah:/blah",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+
+ for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr2->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(itr2->second, ret);
+ ASSERT_STREQ("", buff);
+ }
+
+ // verify that you can't mount any directory in the container-executor.cfg path
+ char *ce_path = realpath("../etc/hadoop/container-executor.cfg", NULL);
+ while (strlen(ce_path) != 0) {
+ std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n rw-mounts=";
+ cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
+ memset(buff, 0, buff_len);
+ write_command_file(cmd_file_contents);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret) << " for input " << cmd_file_contents;
+ ASSERT_STREQ("", buff);
+ char *tmp = strrchr(ce_path, '/');
+ if (tmp != NULL) {
+ *tmp = '\0';
+ }
+ }
+ free(ce_path);
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_add_ro_mounts) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls\n "
+ "docker.allowed.ro-mounts=/bin/cat,/bin/ln";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/var:/var", "-v '/var:/var:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr:/usr", "-v '/usr:/usr:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/:/usr", "-v '/usr/:/usr:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ln:/bin/ln", "-v '/bin/ln:/bin/ln:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/cat:/bin/cat",
+ "-v '/bin/cat:/bin/cat:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/bin:/mydisk1,/bin/cat:/bin/cat",
+ "-v '/usr/bin:/mydisk1:ro' -v '/bin/cat:/bin/cat:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home",
+ static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/blah:/blah",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+
+ for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr2->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(itr2->second, ret);
+ ASSERT_STREQ("", buff);
+ }
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home");
+ strcpy(buff, "test string");
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_docker_run_privileged) {
+
+ std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n"
+ " docker.allowed.devices=/dev/test";
+ write_file(container_executor_cfg_file, container_executor_contents);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+ "'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+ " 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' "
+ "'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' "
+ "--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=docker-image\n user=test",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=test\n",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n",
+ static_cast<int>(INVALID_DOCKER_USER_NAME)));
+
+ // invalid rw mount
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/var/log:/var/log\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+
+ // invalid ro mount
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/bin:/bin,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
+
+ // invalid capability
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_CAPABILITY)));
+
+ // invalid device
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_DEVICE)));
+
+ // invalid network
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_NETWORK)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
+
+ std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.allowed.capabilities=CHOWN,SETUID\n"
+ " docker.allowed.devices=/dev/test",
+ "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.allowed.capabilities=CHOWN,SETUID\n"
+ " privileged=0\n"
+ " docker.allowed.devices=/dev/test"};
+ for (int i = 0; i < 2; ++i) {
+ write_file(container_executor_cfg_file, container_executor_contents[i]);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n"
+ " user=test\n launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+ "'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+ " 'test_script.sh' 'arg1' 'arg2' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(PRIVILEGED_CONTAINERS_DISABLED)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_config_param) {
+ std::vector<std::pair<std::string, std::string> > input_output_map;
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n docker-config=/my-config\n"
+ " format={{.State.Status}}\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' inspect --format={{.State.Status}} container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=load\n docker-config=/my-config\n image=image-id",
+ "--config='/my-config' load --i='image-id' "));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=pull\n docker-config=/my-config\n image=image-id",
+ "--config='/my-config' pull 'image-id' "));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=rm\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' rm container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' stop container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001\n"
+ " image=docker-image\n user=test",
+ "--config='/my-config' run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ char buffer[4096];
+ struct configuration cfg = {0, NULL};
+ for (itr = input_output_map.begin(); itr != input_output_map.end(); ++itr) {
+ memset(buffer, 0, 4096);
+ write_command_file(itr->first);
+ int ret = get_docker_command(docker_command_file.c_str(), &cfg, buffer, 4096);
+ ASSERT_EQ(0, ret) << "for input " << itr->first;
+ ASSERT_STREQ(itr->second.c_str(), buffer);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_module_enabled) {
+
+ std::vector<std::pair<std::string, int> > input_out_vec;
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=true", 1));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=false", 0));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=1", 0));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n", 0));
+
+ for (size_t i = 0; i < input_out_vec.size(); ++i) {
+ write_file(container_executor_cfg_file, input_out_vec[i].first);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = docker_module_enabled(&container_executor_cfg);
+ ASSERT_EQ(input_out_vec[i].second, ret) << " incorrect output for "
+ << input_out_vec[i].first;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index e548790..fbfee54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -295,32 +295,37 @@ public class TestDockerContainerRuntime {
List<String> args = op.getArguments();
String dockerCommandFile = args.get(11);
- //This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
- .append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=host ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ")
- .append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ")
- .append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -347,10 +352,13 @@ public class TestDockerContainerRuntime {
String uid = "";
String gid = "";
+ String[] groups = {};
Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor(
new String[]{"id", "-u", runAsUser});
Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor(
new String[]{"id", "-g", runAsUser});
+ Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-G", runAsUser});
try {
shexec1.execute();
// get rid of newline at the end
@@ -365,37 +373,48 @@ public class TestDockerContainerRuntime {
} catch (Exception e) {
LOG.info("Could not run id -g command: " + e);
}
+ try {
+ shexec3.execute();
+ groups = shexec3.getOutput().replace("\n", " ").split(" ");
+ } catch (Exception e) {
+ LOG.info("Could not run id -G command: " + e);
+ }
uidGidPair = uid + ":" + gid;
- //This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
- .append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=host ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ")
- .append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ")
- .append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ")
- .append("(--group-add \\d+ )*")
- .append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, uidGidPair,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- //Assert.assertEquals(expectedCommand, dockerCommands.get(0));
- Assert.assertTrue(dockerCommands.get(0).matches(expectedCommand));
+ Assert.assertEquals(14, dockerCommands.size());
+ int counter = 0;
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id",
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -481,29 +500,38 @@ public class TestDockerContainerRuntime {
String dockerCommandFile = args.get(11);
//This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + allowedNetwork + " ")
- .append("--hostname=" + expectedHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
-
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=test.hostname",
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" net=" + allowedNetwork, dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -537,30 +565,37 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case. customNetwork1
// ("sdn1") is the expected network to be used in this case
- StringBuffer expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + customNetwork1 + " ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
-
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn1", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
//now set an explicit (non-default) allowedNetwork and ensure that it is
// used.
@@ -575,28 +610,37 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case. customNetwork2
// ("sdn2") is the expected network to be used in this case
- expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + customNetwork2 + " ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+ counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn2", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
//disallowed network should trigger a launch failure
@@ -630,7 +674,8 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
+ int expected = 13;
+ Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -738,13 +783,35 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
-
- //submitting user is whitelisted. ensure --privileged is in the invocation
- Assert.assertTrue("Did not find expected '--privileged' in docker run args "
- + ": " + command, command.contains("--privileged"));
+ int expected = 14;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" privileged=true", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -833,15 +900,33 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
-
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount" +
- ":ro "));
+ Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(1));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
+ Assert.assertEquals(" detach=true", dockerCommands.get(3));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
+ Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(7));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(8));
+ Assert.assertEquals(" net=host", dockerCommands.get(9));
+ Assert.assertEquals(
+ " ro-mounts=/test_local_dir/test_resource_file:test_mount",
+ dockerCommands.get(10));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(11));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(13));
}
@Test
@@ -885,20 +970,35 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
+ Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(1));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
+ Assert.assertEquals(" detach=true", dockerCommands.get(3));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
+ Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(7));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(8));
+ Assert.assertEquals(" net=host", dockerCommands.get(9));
+ Assert.assertEquals(
+ " ro-mounts=/test_local_dir/test_resource_file:test_mount1,"
+ + "/test_local_dir/test_resource_file:test_mount2",
+ dockerCommands.get(10));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(11));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(13));
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount1 mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount1" +
- ":ro "));
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount2 mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount2" +
- ":ro "));
}
@Test
@@ -930,8 +1030,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.TERM);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
@Test
@@ -940,8 +1042,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.KILL);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
@Test
@@ -950,8 +1054,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.QUIT);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
private List<String> getDockerCommandsForSignal(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDFS-12495.
TestPendingInvalidateBlock#testPendingDeleteUnknownBlocks fails
intermittently. Contributed by Eric Badger.
Posted by kk...@apache.org.
HDFS-12495. TestPendingInvalidateBlock#testPendingDeleteUnknownBlocks fails intermittently. Contributed by Eric Badger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9b790db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9b790db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9b790db
Branch: refs/heads/YARN-6592
Commit: e9b790db9de7c3fbe451bcdc3fd928a27a0d8b4e
Parents: 06e5a7b
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Sep 26 17:21:42 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Sep 26 17:21:42 2017 +0800
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/TestPendingInvalidateBlock.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9b790db/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
index 674f883..8f7b5ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java
@@ -172,7 +172,7 @@ public class TestPendingInvalidateBlock {
Assert.assertEquals(0L, cluster.getNamesystem().getPendingDeletionBlocks());
// restart DataNodes
for (int i = 0; i < REPLICATION; i++) {
- cluster.restartDataNode(dnprops[i], true);
+ cluster.restartDataNode(dnprops[i]);
}
cluster.waitActive();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDFS-12552. Use slf4j instead of
log4j in FSNamesystem. Contributed by Ajay Kumar.
Posted by kk...@apache.org.
HDFS-12552. Use slf4j instead of log4j in FSNamesystem. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a68c2ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a68c2ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a68c2ce
Branch: refs/heads/YARN-6592
Commit: 5a68c2cef31031522f2532064965f482017dfefd
Parents: 563dcdf
Author: Chen Liang <cl...@apache.org>
Authored: Mon Oct 2 11:26:53 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Oct 2 11:26:53 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 9 ++++++---
.../org/apache/hadoop/hdfs/TestFileConcurrentReader.java | 8 ++++----
.../java/org/apache/hadoop/hdfs/TestFileCreation.java | 8 ++++----
.../org/apache/hadoop/hdfs/TestFileCreationClient.java | 10 +++++-----
.../test/java/org/apache/hadoop/hdfs/TestFileStatus.java | 6 +++---
.../java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java | 8 ++++----
.../java/org/apache/hadoop/hdfs/TestPersistBlocks.java | 6 +++---
.../java/org/apache/hadoop/hdfs/TestReadWhileWriting.java | 6 +++---
.../blockmanagement/TestBlocksWithNotEnoughRacks.java | 6 +++---
.../hadoop/hdfs/server/namenode/TestFSNamesystemLock.java | 2 +-
10 files changed, 36 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4094694..dc50808 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -310,6 +310,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.LoggerFactory;
/**
* FSNamesystem is a container of both transient
@@ -342,7 +343,9 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
@Metrics(context="dfs")
public class FSNamesystem implements Namesystem, FSNamesystemMBean,
NameNodeMXBean, ReplicatedBlocksMBean, ECBlockGroupsMBean {
- public static final Log LOG = LogFactory.getLog(FSNamesystem.class);
+
+ public static final org.slf4j.Logger LOG = LoggerFactory
+ .getLogger(FSNamesystem.class.getName());
private final MetricsRegistry registry = new MetricsRegistry("FSNamesystem");
@Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics =
registry.newRatesWithAggregation("detailedLockHoldTimeMetrics");
@@ -1671,8 +1674,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
stopStandbyServices();
} catch (IOException ie) {
} finally {
- IOUtils.cleanup(LOG, dir);
- IOUtils.cleanup(LOG, fsImage);
+ IOUtils.cleanupWithLogger(LOG, dir);
+ IOUtils.cleanupWithLogger(LOG, fsImage);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
index e0a0d3b..7557197 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
@@ -38,12 +38,12 @@ import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
@@ -62,9 +62,9 @@ public class TestFileConcurrentReader {
Logger.getLogger(TestFileConcurrentReader.class);
{
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(DFSClient.LOG, Level.TRACE);
}
static final long seed = 0xDEADBEEFL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index e191414..a738343 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -86,9 +86,9 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
-import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* This class tests various cases during file creation.
@@ -97,9 +97,9 @@ public class TestFileCreation {
static final String DIR = "/" + TestFileCreation.class.getSimpleName() + "/";
{
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(DFSClient.LOG, Level.TRACE);
}
private static final String RPC_DETAILED_METRICS =
"RpcDetailedActivityForPort";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
index db291a0..986bb56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationClient.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* This class tests client lease recovery.
@@ -40,10 +40,10 @@ public class TestFileCreationClient {
static final String DIR = "/" + TestFileCreationClient.class.getSimpleName() + "/";
{
- GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(DataNode.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(InterDatanodeProtocol.LOG, Level.TRACE);
}
/** Test lease recovery Triggered by DFSClient. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
index 31007dd..a5f8911 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
@@ -38,18 +38,18 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* This class tests the FileStatus API.
*/
public class TestFileStatus {
{
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FileSystem.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FileSystem.LOG, Level.TRACE);
}
static final long seed = 0xDEADBEEFL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
index a807bbb..a96d8b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
@@ -52,21 +52,21 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import org.slf4j.event.Level;
public class TestLeaseRecovery2 {
public static final Log LOG = LogFactory.getLog(TestLeaseRecovery2.class);
{
- GenericTestUtils.setLogLevel(DataNode.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(DataNode.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(LeaseManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
}
static final private long BLOCK_SIZE = 1024;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
index 54eb934..4a99493 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPersistBlocks.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
-import org.apache.log4j.Level;
import org.junit.Test;
+import org.slf4j.event.Level;
/**
* A JUnit test for checking if restarting DFS preserves the
@@ -52,8 +52,8 @@ import org.junit.Test;
*/
public class TestPersistBlocks {
static {
- GenericTestUtils.setLogLevel(FSImage.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSImage.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
}
private static final int BLOCK_SIZE = 4096;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
index 2b55d25..8de1799 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadWhileWriting.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.event.Level;
/** Test reading from hdfs while a file is being written. */
public class TestReadWhileWriting {
{
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(DFSClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(DFSClient.LOG, Level.TRACE);
}
private static final String DIR = "/"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
index 0298cbc..2bf6045 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java
@@ -40,14 +40,14 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.util.HostsFileWriter;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.junit.Test;
+import org.slf4j.event.Level;
public class TestBlocksWithNotEnoughRacks {
public static final Log LOG = LogFactory.getLog(TestBlocksWithNotEnoughRacks.class);
static {
- GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(LOG, Level.TRACE);
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a68c2ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
index 94e0b33..2daf5c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemLock.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.test.MetricsAsserts;
import org.apache.hadoop.util.FakeTimer;
-import org.apache.log4j.Level;
import org.junit.Test;
import java.io.IOException;
@@ -39,6 +38,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
import static org.junit.Assert.*;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FSLOCK_FAIR_KEY;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HADOOP-14095. Document caveats
about the default JavaKeyStoreProvider in KMS.
Posted by kk...@apache.org.
HADOOP-14095. Document caveats about the default JavaKeyStoreProvider in KMS.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06df6ab2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06df6ab2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06df6ab2
Branch: refs/heads/YARN-6592
Commit: 06df6ab25432927cd55918440cc8a28fb4f03bfa
Parents: 66c4171
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Sep 29 19:17:32 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Sep 29 19:34:29 2017 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06df6ab2/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 1dd89e9..5490219 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -80,6 +80,8 @@ The password file is looked up in the Hadoop's configuration directory via the c
NOTE: You need to restart the KMS for the configuration changes to take effect.
+NOTE: The KMS server can choose any `KeyProvider` implementation as the backing provider. The example here uses a JavaKeyStoreProvider, which should only be used for experimental purposes and never be used in production. For detailed usage and caveats of JavaKeyStoreProvider, please see [Keystore Passwords section of the Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html#Keystore_Passwords).
+
$H3 KMS HTTP Configuration
KMS pre-configures the HTTP port to 9600.
@@ -1184,4 +1186,4 @@ and `/stacks`, configure the following properties in `kms-site.xml`:
to all users and groups, e.g. '*', '* ' and ' *' are all valid.
</description>
</property>
-```
\ No newline at end of file
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-6550. Capture
launch_container.sh logs to a separate log file. (Suma Shivaprasad via
wangda)
Posted by kk...@apache.org.
YARN-6550. Capture launch_container.sh logs to a separate log file. (Suma Shivaprasad via wangda)
Change-Id: I0ee0b1bb459437432a22cf68861a6354f0decabb
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/febeead5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/febeead5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/febeead5
Branch: refs/heads/YARN-6592
Commit: febeead5f95c6fc245ea3735f5b538d4bb4dc8a4
Parents: 373d0a5
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Sep 29 16:39:46 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Sep 29 16:39:46 2017 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 20 +-
.../nodemanager/DefaultContainerExecutor.java | 3 +-
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../launcher/ContainerLaunch.java | 189 ++++++++++++++++---
.../launcher/TestContainerLaunch.java | 115 ++++++++++-
5 files changed, 290 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index da50d7a..5fd059d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -64,6 +64,9 @@ import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT;
+
/**
* This class is abstraction of the mechanism used to launch a container on the
* underlying OS. All executor implementations must extend ContainerExecutor.
@@ -330,6 +333,14 @@ public abstract class ContainerExecutor implements Configurable {
String user, String outFilename) throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
+
+ // Add "set -o pipefail -e" to validate launch_container script.
+ sb.setExitOnFailure();
+
+ //Redirect stdout and stderr for launch_container script
+ sb.stdout(logDir, CONTAINER_PRE_LAUNCH_STDOUT);
+ sb.stderr(logDir, CONTAINER_PRE_LAUNCH_STDERR);
+
Set<String> whitelist = new HashSet<>();
String[] nmWhiteList = conf.get(YarnConfiguration.NM_ENV_WHITELIST,
@@ -338,10 +349,8 @@ public abstract class ContainerExecutor implements Configurable {
whitelist.add(param);
}
- // Add "set -o pipefail -e" to validate launch_container script.
- sb.setExitOnFailure();
-
if (environment != null) {
+ sb.echo("Setting up env variables");
for (Map.Entry<String, String> env : environment.entrySet()) {
if (!whitelist.contains(env.getKey())) {
sb.env(env.getKey(), env.getValue());
@@ -352,6 +361,7 @@ public abstract class ContainerExecutor implements Configurable {
}
if (resources != null) {
+ sb.echo("Setting up job resources");
for (Map.Entry<Path, List<String>> resourceEntry :
resources.entrySet()) {
for (String linkName : resourceEntry.getValue()) {
@@ -373,15 +383,15 @@ public abstract class ContainerExecutor implements Configurable {
if (getConf() != null &&
getConf().getBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO,
YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO)) {
+ sb.echo("Copying debugging information");
sb.copyDebugInformation(new Path(outFilename),
new Path(logDir, outFilename));
sb.listDebugInformation(new Path(logDir, DIRECTORY_CONTENTS));
}
-
+ sb.echo("Launching container");
sb.command(command);
PrintStream pout = null;
-
try {
pout = new PrintStream(out, false, "UTF-8");
sb.write(pout);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 8c58056..2b32234 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -320,8 +320,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
builder.append("Exception message: ");
builder.append(e.getMessage()).append("\n");
}
- builder.append("Stack trace: ");
- builder.append(StringUtils.stringifyException(e)).append("\n");
+
if (!shExec.getOutput().isEmpty()) {
builder.append("Shell output: ");
builder.append(shExec.getOutput()).append("\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 2971f83..9e004e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -532,8 +532,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
if (!Optional.fromNullable(e.getErrorOutput()).or("").isEmpty()) {
builder.append("Exception message: " + e.getErrorOutput() + "\n");
}
- builder.append("Stack trace: "
- + StringUtils.stringifyException(e) + "\n");
+ //Skip stack trace
String output = e.getOutput();
if (output != null && !e.getOutput().isEmpty()) {
builder.append("Shell output: " + output + "\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index e254887..f929dfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -87,6 +87,8 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -94,8 +96,13 @@ public class ContainerLaunch implements Callable<Integer> {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerLaunch.class);
+ private static final String CONTAINER_PRE_LAUNCH_PREFIX = "prelaunch";
+ public static final String CONTAINER_PRE_LAUNCH_STDOUT = CONTAINER_PRE_LAUNCH_PREFIX + ".out";
+ public static final String CONTAINER_PRE_LAUNCH_STDERR = CONTAINER_PRE_LAUNCH_PREFIX + ".err";
+
public static final String CONTAINER_SCRIPT =
Shell.appendScriptExtension("launch_container");
+
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
private static final String PID_FILE_NAME_FMT = "%s.pid";
@@ -147,7 +154,7 @@ public class ContainerLaunch implements Callable<Integer> {
Path containerLogDir) {
var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
containerLogDir.toString());
- var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
+ var = var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
File.pathSeparator);
// replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
@@ -371,7 +378,7 @@ public class ContainerLaunch implements Callable<Integer> {
String relativeContainerLogDir = ContainerLaunch
.getRelativeContainerLogDir(appIdStr, containerIdStr);
- for(String logDir : logDirs) {
+ for (String logDir : logDirs) {
containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
}
@@ -516,6 +523,7 @@ public class ContainerLaunch implements Callable<Integer> {
* Tries to tail and fetch TAIL_SIZE_IN_BYTES of data from the error log.
* ErrorLog filename is not fixed and depends upon app, hence file name
* pattern is used.
+ *
* @param containerID
* @param ret
* @param containerLogDir
@@ -524,20 +532,46 @@ public class ContainerLaunch implements Callable<Integer> {
@SuppressWarnings("unchecked")
protected void handleContainerExitWithFailure(ContainerId containerID,
int ret, Path containerLogDir, StringBuilder diagnosticInfo) {
- LOG.warn(diagnosticInfo.toString());
+ LOG.warn("Container launch failed : " + diagnosticInfo.toString());
+
+ FileSystem fileSystem = null;
+ long tailSizeInBytes =
+ conf.getLong(YarnConfiguration.NM_CONTAINER_STDERR_BYTES,
+ YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_BYTES);
+
+ // Append container prelaunch stderr to diagnostics
+ try {
+ fileSystem = FileSystem.getLocal(conf).getRaw();
+ FileStatus preLaunchErrorFileStatus = fileSystem
+ .getFileStatus(new Path(containerLogDir, ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR));
+
+ Path errorFile = preLaunchErrorFileStatus.getPath();
+ long fileSize = preLaunchErrorFileStatus.getLen();
+
+ diagnosticInfo.append("Error file: ")
+ .append(ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR).append(".\n");
+ ;
+ byte[] tailBuffer = tailFile(errorFile, fileSize, tailSizeInBytes);
+ diagnosticInfo.append("Last ").append(tailSizeInBytes)
+ .append(" bytes of ").append(errorFile.getName()).append(" :\n")
+ .append(new String(tailBuffer, StandardCharsets.UTF_8));
+ } catch (IOException e) {
+ LOG.error("Failed to get tail of the container's prelaunch error log file", e);
+ }
+
+ // Append container stderr to diagnostics
String errorFileNamePattern =
conf.get(YarnConfiguration.NM_CONTAINER_STDERR_PATTERN,
YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_PATTERN);
- FSDataInputStream errorFileIS = null;
+
try {
- FileSystem fileSystem = FileSystem.getLocal(conf).getRaw();
+ if (fileSystem == null) {
+ fileSystem = FileSystem.getLocal(conf).getRaw();
+ }
FileStatus[] errorFileStatuses = fileSystem
.globStatus(new Path(containerLogDir, errorFileNamePattern));
if (errorFileStatuses != null && errorFileStatuses.length != 0) {
- long tailSizeInBytes =
- conf.getLong(YarnConfiguration.NM_CONTAINER_STDERR_BYTES,
- YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_BYTES);
Path errorFile = errorFileStatuses[0].getPath();
long fileSize = errorFileStatuses[0].getLen();
@@ -560,32 +594,40 @@ public class ContainerLaunch implements Callable<Integer> {
.append(StringUtils.join(", ", errorFileNames)).append(".\n");
}
- long startPosition =
- (fileSize < tailSizeInBytes) ? 0 : fileSize - tailSizeInBytes;
- int bufferSize =
- (int) ((fileSize < tailSizeInBytes) ? fileSize : tailSizeInBytes);
- byte[] tailBuffer = new byte[bufferSize];
- errorFileIS = fileSystem.open(errorFile);
- errorFileIS.readFully(startPosition, tailBuffer);
-
+ byte[] tailBuffer = tailFile(errorFile, fileSize, tailSizeInBytes);
String tailBufferMsg = new String(tailBuffer, StandardCharsets.UTF_8);
diagnosticInfo.append("Last ").append(tailSizeInBytes)
.append(" bytes of ").append(errorFile.getName()).append(" :\n")
.append(tailBufferMsg).append("\n")
.append(analysesErrorMsgOfContainerExitWithFailure(tailBufferMsg));
+
}
} catch (IOException e) {
LOG.error("Failed to get tail of the container's error log file", e);
- } finally {
- IOUtils.cleanupWithLogger(LOG, errorFileIS);
}
-
this.dispatcher.getEventHandler()
.handle(new ContainerExitEvent(containerID,
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
diagnosticInfo.toString()));
}
+ private byte[] tailFile(Path filePath, long fileSize, long tailSizeInBytes) throws IOException {
+ FSDataInputStream errorFileIS = null;
+ FileSystem fileSystem = FileSystem.getLocal(conf).getRaw();
+ try {
+ long startPosition =
+ (fileSize < tailSizeInBytes) ? 0 : fileSize - tailSizeInBytes;
+ int bufferSize =
+ (int) ((fileSize < tailSizeInBytes) ? fileSize : tailSizeInBytes);
+ byte[] tailBuffer = new byte[bufferSize];
+ errorFileIS = fileSystem.open(filePath);
+ errorFileIS.readFully(startPosition, tailBuffer);
+ return tailBuffer;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, errorFileIS);
+ }
+ }
+
private String analysesErrorMsgOfContainerExitWithFailure(String errorMsg) {
StringBuilder analysis = new StringBuilder();
if (errorMsg.indexOf("Error: Could not find or load main class"
@@ -982,8 +1024,48 @@ public class ContainerLaunch implements Callable<Integer> {
public abstract void whitelistedEnv(String key, String value) throws IOException;
+ protected static final String ENV_PRELAUNCH_STDOUT = "PRELAUNCH_OUT";
+ protected static final String ENV_PRELAUNCH_STDERR = "PRELAUNCH_ERR";
+
+ private boolean redirectStdOut = false;
+ private boolean redirectStdErr = false;
+
+ /**
+ * Set stdout for the shell script
+ * @param stdoutDir stdout must be an absolute path
+ * @param stdOutFile stdout file name
+ * @throws IOException thrown when stdout path is not absolute
+ */
+ public final void stdout(Path stdoutDir, String stdOutFile) throws IOException {
+ if (!stdoutDir.isAbsolute()) {
+ throw new IOException("Stdout path must be absolute");
+ }
+ redirectStdOut = true;
+ setStdOut(new Path(stdoutDir, stdOutFile));
+ }
+
+ /**
+ * Set stderr for the shell script
+ * @param stderrDir stderr must be an absolute path
+ * @param stdErrFile stderr file name
+ * @throws IOException thrown when stderr path is not absolute
+ */
+ public final void stderr(Path stderrDir, String stdErrFile) throws IOException {
+ if (!stderrDir.isAbsolute()) {
+ throw new IOException("Stdout path must be absolute");
+ }
+ redirectStdErr = true;
+ setStdErr(new Path(stderrDir, stdErrFile));
+ }
+
+ protected abstract void setStdOut(Path stdout) throws IOException;
+
+ protected abstract void setStdErr(Path stdout) throws IOException;
+
public abstract void env(String key, String value) throws IOException;
+ public abstract void echo(String echoStr) throws IOException;
+
public final void symlink(Path src, Path dst) throws IOException {
if (!src.isAbsolute()) {
throw new IOException("Source must be absolute");
@@ -1028,13 +1110,21 @@ public class ContainerLaunch implements Callable<Integer> {
out.append(sb);
}
- protected final void line(String... command) {
+ protected final void buildCommand(String... command) {
for (String s : command) {
sb.append(s);
}
+ }
+
+ protected final void linebreak(String... command) {
sb.append(LINE_SEPARATOR);
}
+ protected final void line(String... command) {
+ buildCommand(command);
+ linebreak();
+ }
+
public void setExitOnFailure() {
// Dummy implementation
}
@@ -1042,19 +1132,27 @@ public class ContainerLaunch implements Callable<Integer> {
protected abstract void link(Path src, Path dst) throws IOException;
protected abstract void mkdir(Path path) throws IOException;
+
+ boolean doRedirectStdOut() {
+ return redirectStdOut;
+ }
+
+ boolean doRedirectStdErr() {
+ return redirectStdErr;
+ }
+
}
private static final class UnixShellScriptBuilder extends ShellScriptBuilder {
-
private void errorCheck() {
line("hadoop_shell_errorcode=$?");
- line("if [ $hadoop_shell_errorcode -ne 0 ]");
+ line("if [[ \"$hadoop_shell_errorcode\" -ne 0 ]]");
line("then");
line(" exit $hadoop_shell_errorcode");
line("fi");
}
- public UnixShellScriptBuilder(){
+ public UnixShellScriptBuilder() {
line("#!/bin/bash");
line();
}
@@ -1062,29 +1160,47 @@ public class ContainerLaunch implements Callable<Integer> {
@Override
public void command(List<String> command) {
line("exec /bin/bash -c \"", StringUtils.join(" ", command), "\"");
- errorCheck();
}
@Override
- public void whitelistedEnv(String key, String value) {
+ public void whitelistedEnv(String key, String value) throws IOException {
line("export ", key, "=${", key, ":-", "\"", value, "\"}");
}
@Override
- public void env(String key, String value) {
+ public void setStdOut(final Path stdout) throws IOException {
+ line("export ", ENV_PRELAUNCH_STDOUT, "=\"", stdout.toString(), "\"");
+ // tee is needed for DefaultContainerExecutor error propagation to stdout
+ // Close stdout of subprocess to prevent it from writing to the stdout file
+ line("exec >\"${" + ENV_PRELAUNCH_STDOUT + "}\"");
+ }
+
+ @Override
+ public void setStdErr(final Path stderr) throws IOException {
+ line("export ", ENV_PRELAUNCH_STDERR, "=\"", stderr.toString(), "\"");
+ // tee is needed for DefaultContainerExecutor error propagation to stderr
+ // Close stdout of subprocess to prevent it from writing to the stdout file
+ line("exec 2>\"${" + ENV_PRELAUNCH_STDERR + "}\"");
+ }
+
+ @Override
+ public void env(String key, String value) throws IOException {
line("export ", key, "=\"", value, "\"");
}
@Override
+ public void echo(final String echoStr) throws IOException {
+ line("echo \"" + echoStr + "\"");
+ }
+
+ @Override
protected void link(Path src, Path dst) throws IOException {
line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\"");
- errorCheck();
}
@Override
- protected void mkdir(Path path) {
+ protected void mkdir(Path path) throws IOException {
line("mkdir -p ", path.toString());
- errorCheck();
}
@Override
@@ -1152,6 +1268,16 @@ public class ContainerLaunch implements Callable<Integer> {
errorCheck();
}
+ //Dummy implementation
+ @Override
+ protected void setStdOut(final Path stdout) throws IOException {
+ }
+
+ //Dummy implementation
+ @Override
+ protected void setStdErr(final Path stderr) throws IOException {
+ }
+
@Override
public void env(String key, String value) throws IOException {
lineWithLenCheck("@set ", key, "=", value);
@@ -1159,6 +1285,11 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
+ public void echo(final String echoStr) throws IOException {
+ lineWithLenCheck("@echo \"", echoStr, "\"");
+ }
+
+ @Override
protected void link(Path src, Path dst) throws IOException {
File srcFile = new File(src.toUri().getPath());
String srcFileStr = srcFile.getPath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/febeead5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 7176942..588fdcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -192,7 +193,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
shexc.execute();
assertEquals(shexc.getExitCode(), 0);
- assert(shexc.getOutput().contains("hello"));
+ //Capture output from prelaunch.out
+
+ List<String> output = Files.readAllLines(Paths.get(localLogDir.getAbsolutePath(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT),
+ Charset.forName("UTF-8"));
+ assert(output.contains("hello"));
symLinkFile = new File(tmpDir, badSymlink);
}
@@ -358,7 +363,10 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
shexc.execute();
Assert.fail("Should catch exception");
} catch(ExitCodeException e){
- diagnostics = e.getMessage();
+ //Capture diagnostics from prelaunch.stderr
+ List<String> error = Files.readAllLines(Paths.get(localLogDir.getAbsolutePath(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR),
+ Charset.forName("UTF-8"));
+ diagnostics = StringUtils.join("\n", error);
}
Assert.assertTrue(diagnostics.contains(Shell.WINDOWS ?
"is not recognized as an internal or external command" :
@@ -1545,6 +1553,109 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
+ /**
+ * Test that script exists with non-zero exit code when command fails.
+ * @throws IOException
+ */
+ @Test
+ public void testShellScriptBuilderStdOutandErrRedirection() throws IOException {
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ Path logDir = new Path(localLogDir.getAbsolutePath());
+ File stdout = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT);
+ File stderr = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR);
+
+ builder.stdout(logDir, ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT);
+ builder.stderr(logDir, ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR);
+
+ //should redirect to specified stdout path
+ String TEST_STDOUT_ECHO = "Test stdout redirection";
+ builder.echo(TEST_STDOUT_ECHO);
+ //should fail and redirect to stderr
+ builder.mkdir(new Path("/invalidSrcDir"));
+
+ builder.command(Arrays.asList(new String[] {"unknownCommand"}));
+
+ File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderStdOutandErrRedirection");
+ PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
+ builder.write(writer);
+ writer.close();
+ try {
+ FileUtil.setExecutable(shellFile, true);
+
+ Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
+ new String[]{shellFile.getAbsolutePath()}, tmpDir);
+ try {
+ shexc.execute();
+ fail("builder shell command was expected to throw");
+ }
+ catch(IOException e) {
+ // expected
+ System.out.println("Received an expected exception: " + e.getMessage());
+
+ Assert.assertEquals(true, stdout.exists());
+ BufferedReader stdoutReader = new BufferedReader(new FileReader(stdout));
+ // Get the pid of the process
+ String line = stdoutReader.readLine().trim();
+ Assert.assertEquals(TEST_STDOUT_ECHO, line);
+ // No more lines
+ Assert.assertEquals(null, stdoutReader.readLine());
+ stdoutReader.close();
+
+ Assert.assertEquals(true, stderr.exists());
+ Assert.assertTrue(stderr.length() > 0);
+ }
+ }
+ finally {
+ FileUtil.fullyDelete(shellFile);
+ FileUtil.fullyDelete(stdout);
+ FileUtil.fullyDelete(stderr);
+ }
+ }
+
+ /**
+ * Test that script exists with non-zero exit code when command fails.
+ * @throws IOException
+ */
+ @Test
+ public void testShellScriptBuilderWithNoRedirection() throws IOException {
+ ShellScriptBuilder builder = ShellScriptBuilder.create();
+
+ Path logDir = new Path(localLogDir.getAbsolutePath());
+ File stdout = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDOUT);
+ File stderr = new File(logDir.toString(), ContainerLaunch.CONTAINER_PRE_LAUNCH_STDERR);
+
+ //should redirect to specified stdout path
+ String TEST_STDOUT_ECHO = "Test stdout redirection";
+ builder.echo(TEST_STDOUT_ECHO);
+ //should fail and redirect to stderr
+ builder.mkdir(new Path("/invalidSrcDir"));
+
+ builder.command(Arrays.asList(new String[]{"unknownCommand"}));
+
+ File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderStdOutandErrRedirection");
+ PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
+ builder.write(writer);
+ writer.close();
+ try {
+ FileUtil.setExecutable(shellFile, true);
+
+ Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
+ new String[]{shellFile.getAbsolutePath()}, tmpDir);
+ try {
+ shexc.execute();
+ fail("builder shell command was expected to throw");
+ } catch (IOException e) {
+ // expected
+ System.out.println("Received an expected exception: " + e.getMessage());
+
+ Assert.assertEquals(false, stdout.exists());
+ Assert.assertEquals(false, stderr.exists());
+ }
+ } finally {
+ FileUtil.fullyDelete(shellFile);
+ }
+ }
/*
* ${foo.version} is substituted to suffix a specific version number
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-7248. NM returns new
SCHEDULED container status to older clients. Contributed by Arun Suresh
Posted by kk...@apache.org.
YARN-7248. NM returns new SCHEDULED container status to older clients. Contributed by Arun Suresh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85d81ae5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85d81ae5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85d81ae5
Branch: refs/heads/YARN-6592
Commit: 85d81ae58ec4361a944c84753a900460a0888b9b
Parents: a530e7a
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 28 14:10:15 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 28 14:10:15 2017 -0500
----------------------------------------------------------------------
.../hadoop/yarn/api/records/ContainerState.java | 10 +--
.../yarn/api/records/ContainerStatus.java | 22 ++++++
.../yarn/api/records/ContainerSubState.java | 57 ++++++++++++++
.../src/main/proto/yarn_protos.proto | 34 ++++++++-
.../records/impl/pb/ContainerStatusPBImpl.java | 24 +++++-
.../yarn/api/records/impl/pb/ProtoUtils.java | 20 ++++-
.../container/ContainerImpl.java | 33 ++++++++-
.../container/ContainerState.java | 9 +++
.../yarn/server/nodemanager/TestEventFlow.java | 2 +-
.../nodemanager/TestNodeManagerShutdown.java | 3 +-
.../TestContainerSchedulerQueuing.java | 78 ++++++++------------
.../resourcemanager/rmnode/RMNodeImpl.java | 6 +-
.../TestResourceTrackerService.java | 2 +-
13 files changed, 229 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
index 45e5bd4..e22204e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Stable;
/**
@@ -35,12 +34,5 @@ public enum ContainerState {
RUNNING,
/** Completed container */
- COMPLETE,
-
- /** Scheduled (awaiting resources) at the NM. */
- @InterfaceStability.Unstable
- SCHEDULED,
-
- /** Paused at the NM. */
- PAUSED
+ COMPLETE
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
index d7c75f3..edc62fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
@@ -201,4 +201,26 @@ public abstract class ContainerStatus {
throw new UnsupportedOperationException(
"subclass must implement this method");
}
+
+ /**
+ * Add Extra state information of the container (SCHEDULED, LOCALIZING etc.).
+ * @param subState Extra State Information.
+ */
+ @Private
+ @Unstable
+ public void setContainerSubState(ContainerSubState subState) {
+ throw new UnsupportedOperationException(
+ "subclass must implement this method");
+ }
+
+ /**
+ * Get Extra state information of the container (SCHEDULED, LOCALIZING etc.).
+ * @return Extra State information.
+ */
+ @Private
+ @Unstable
+ public ContainerSubState getContainerSubState() {
+ throw new UnsupportedOperationException(
+ "subclass must implement this method");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java
new file mode 100644
index 0000000..fed4537
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerSubState.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Container Sub-State.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum ContainerSubState {
+ /*
+ * NEW, LOCALIZING, SCHEDULED,
+ * REINITIALIZING_AWAITING_KILL, RELAUNCHING,
+ */
+ SCHEDULED,
+
+ /*
+ * RUNNING, REINITIALIZING, PAUSING, KILLING
+ */
+ RUNNING,
+
+ /*
+ * PAUSED, RESUMING
+ */
+ PAUSED,
+
+ /*
+ * LOCALIZATION_FAILED, EXITED_WITH_SUCCESS,
+ * EXITED_WITH_FAILURE,
+ * CONTAINER_CLEANEDUP_AFTER_KILL,
+ * CONTAINER_RESOURCES_CLEANINGUP
+ */
+ COMPLETING,
+
+ /*
+ * DONE
+ */
+ DONE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index fb340d1..e69c07b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -109,8 +109,37 @@ enum ContainerStateProto {
C_NEW = 1;
C_RUNNING = 2;
C_COMPLETE = 3;
- C_SCHEDULED = 4;
- C_PAUSED = 5;
+}
+
+enum ContainerSubStateProto {
+ /**
+ * NEW, LOCALIZING, SCHEDULED,
+ * REINITIALIZING_AWAITING_KILL, RELAUNCHING,
+ */
+ CSS_SCHEDULED = 1;
+
+ /**
+ * RUNNING, REINITIALIZING, PAUSING, KILLING
+ */
+ CSS_RUNNING = 2;
+
+ /**
+ * PAUSED, RESUMING
+ */
+ CSS_PAUSED = 3;
+
+ /**
+ * LOCALIZATION_FAILED, EXITED_WITH_SUCCESS,
+ * EXITED_WITH_FAILURE,
+ * CONTAINER_CLEANEDUP_AFTER_KILL,
+ * CONTAINER_RESOURCES_CLEANINGUP
+ */
+ CSS_COMPLETING = 4;
+
+ /**
+ * DONE
+ */
+ CSS_DONE = 5;
}
message ContainerProto {
@@ -616,6 +645,7 @@ message ContainerStatusProto {
optional ResourceProto capability = 5;
optional ExecutionTypeProto executionType = 6 [default = GUARANTEED];
repeated StringStringMapProto container_attributes = 7;
+ optional ContainerSubStateProto container_sub_state = 8;
}
enum ContainerExitStatusProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index 11a3d86..a6668da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos;
@@ -96,7 +97,8 @@ public class ContainerStatusPBImpl extends ContainerStatus {
sb.append("Diagnostics: ").append(getDiagnostics()).append(", ");
sb.append("ExitStatus: ").append(getExitStatus()).append(", ");
sb.append("IP: ").append(getIPs()).append(", ");
- sb.append("Host: ").append(getHost());
+ sb.append("Host: ").append(getHost()).append(", ");
+ sb.append("ContainerSubState: ").append(getContainerSubState());
sb.append("]");
return sb.toString();
}
@@ -213,6 +215,26 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
builder.setState(convertToProtoFormat(state));
}
+
+ @Override
+ public synchronized ContainerSubState getContainerSubState() {
+ ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasContainerSubState()) {
+ return null;
+ }
+ return ProtoUtils.convertFromProtoFormat(p.getContainerSubState());
+ }
+
+ @Override
+ public synchronized void setContainerSubState(ContainerSubState subState) {
+ maybeInitBuilder();
+ if (subState == null) {
+ builder.clearContainerSubState();
+ return;
+ }
+ builder.setContainerSubState(ProtoUtils.convertToProtoFormat(subState));
+ }
+
@Override
public synchronized ContainerId getContainerId() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 158c2ae..528cf8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.ExecutionType;
@@ -61,6 +62,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportPro
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationTimeoutTypeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerSubStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto;
@@ -92,7 +94,7 @@ public class ProtoUtils {
/*
* ContainerState
*/
- private static String CONTAINER_STATE_PREFIX = "C_";
+ private final static String CONTAINER_STATE_PREFIX = "C_";
public static ContainerStateProto convertToProtoFormat(ContainerState e) {
return ContainerStateProto.valueOf(CONTAINER_STATE_PREFIX + e.name());
}
@@ -101,9 +103,23 @@ public class ProtoUtils {
}
/*
+ * Container SubState
+ */
+ private final static String CONTAINER_SUB_STATE_PREFIX = "CSS_";
+ public static ContainerSubStateProto convertToProtoFormat(
+ ContainerSubState e) {
+ return ContainerSubStateProto.valueOf(
+ CONTAINER_SUB_STATE_PREFIX + e.name());
+ }
+ public static ContainerSubState convertFromProtoFormat(
+ ContainerSubStateProto e) {
+ return ContainerSubState.valueOf(
+ e.name().substring(CONTAINER_SUB_STATE_PREFIX.length()));
+ }
+ /*
* NodeState
*/
- private static String NODE_STATE_PREFIX = "NS_";
+ private final static String NODE_STATE_PREFIX = "NS_";
public static NodeStateProto convertToProtoFormat(NodeState e) {
return NodeStateProto.valueOf(NODE_STATE_PREFIX + e.name());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 2bbc267..df83789 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -34,6 +34,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -714,7 +715,6 @@ public class ContainerImpl implements Container {
case SCHEDULED:
case PAUSED:
case RESUMING:
- return org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED;
case RUNNING:
case RELAUNCHING:
case REINITIALIZING:
@@ -732,6 +732,36 @@ public class ContainerImpl implements Container {
}
}
+ // NOTE: Please update the doc in the ContainerSubState class as
+ // well as the yarn_protos.proto file if this mapping is ever modified.
+ private ContainerSubState getContainerSubState() {
+ switch (stateMachine.getCurrentState()) {
+ case NEW:
+ case LOCALIZING:
+ case SCHEDULED:
+ case REINITIALIZING_AWAITING_KILL:
+ case RELAUNCHING:
+ return ContainerSubState.SCHEDULED;
+ case REINITIALIZING:
+ case PAUSING:
+ case KILLING:
+ case RUNNING:
+ return ContainerSubState.RUNNING;
+ case PAUSED:
+ case RESUMING:
+ return ContainerSubState.PAUSED;
+ case LOCALIZATION_FAILED:
+ case EXITED_WITH_SUCCESS:
+ case EXITED_WITH_FAILURE:
+ case CONTAINER_CLEANEDUP_AFTER_KILL:
+ case CONTAINER_RESOURCES_CLEANINGUP:
+ return ContainerSubState.COMPLETING;
+ case DONE:
+ default:
+ return ContainerSubState.DONE;
+ }
+ }
+
public NMTimelinePublisher getNMTimelinePublisher() {
return context.getNMTimelinePublisher();
}
@@ -800,6 +830,7 @@ public class ContainerImpl implements Container {
this.containerTokenIdentifier.getExecutionType());
status.setIPs(ips == null ? null : Arrays.asList(ips.split(",")));
status.setHost(host);
+ status.setContainerSubState(getContainerSubState());
return status;
} finally {
this.readLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
index 5644d03..a0db646 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
@@ -18,7 +18,16 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+/**
+ * States used by the container state machine.
+ */
public enum ContainerState {
+ // NOTE: In case of future additions / deletions / modifications to this
+ // enum, please ensure that the following are also correspondingly
+ // updated:
+ // 1. ContainerImpl::getContainerSubState().
+ // 2. the doc in the ContainerSubState class.
+ // 3. the doc in the yarn_protos.proto file.
NEW, LOCALIZING, LOCALIZATION_FAILED, SCHEDULED, RUNNING, RELAUNCHING,
REINITIALIZING, REINITIALIZING_AWAITING_KILL,
EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
index 8e4522b..54e090a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
@@ -159,7 +159,7 @@ public class TestEventFlow {
containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
- Arrays.asList(ContainerState.RUNNING, ContainerState.SCHEDULED), 20);
+ Arrays.asList(ContainerState.RUNNING), 20);
List<ContainerId> containerIds = new ArrayList<ContainerId>();
containerIds.add(cID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
index 03e06d2..25dbc1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
@@ -256,8 +256,7 @@ public class TestNodeManagerShutdown {
GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus =
containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
- Assert.assertTrue(
- EnumSet.of(ContainerState.RUNNING, ContainerState.SCHEDULED)
+ Assert.assertTrue(EnumSet.of(ContainerState.RUNNING)
.contains(containerStatus.getState()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 4b380ff..ed4f290 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -311,9 +312,8 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
List<ContainerStatus> containerStatuses = containerManager
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
ContainerScheduler containerScheduler =
@@ -378,13 +378,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
if (status.getContainerId().equals(createContainerId(0))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
} else {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
}
@@ -458,17 +456,15 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
if (status.getContainerId().equals(createContainerId(0))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
} else if (status.getContainerId().equals(createContainerId(
maxOppQueueLength + 1))) {
Assert.assertTrue(status.getDiagnostics().contains(
"Opportunistic container queue is full"));
} else {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
System.out.println("\nStatus : [" + status + "]\n");
}
@@ -545,13 +541,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
Assert.assertTrue(status.getDiagnostics().contains(
"Container Killed to make room for Guaranteed Container"));
} else if (status.getContainerId().equals(createContainerId(1))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
} else if (status.getContainerId().equals(createContainerId(2))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
}
System.out.println("\nStatus : [" + status + "]\n");
}
@@ -799,8 +793,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
"Container De-queued to meet NM queuing limits")) {
deQueuedContainers++;
}
- if (status.getState() ==
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) {
+ if (ContainerSubState.SCHEDULED == status.getContainerSubState()) {
numQueuedOppContainers++;
}
}
@@ -1079,11 +1072,9 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
List<ContainerStatus> containerStatuses = containerManager
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
- if (status.getState() ==
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) {
+ if (ContainerSubState.RUNNING == status.getContainerSubState()) {
runningContainersNo++;
- } else if (status.getState() ==
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) {
+ } else if (ContainerSubState.SCHEDULED == status.getContainerSubState()) {
queuedContainersNo++;
}
System.out.println("\nStatus : [" + status + "]\n");
@@ -1106,34 +1097,27 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
}
statRequest = GetContainerStatusesRequest.newInstance(statList);
- HashMap<org.apache.hadoop.yarn.api.records.ContainerState, ContainerStatus>
- map = new HashMap<>();
+ HashMap<ContainerSubState, ContainerStatus> map = new HashMap<>();
for (int i=0; i < 10; i++) {
containerStatuses = containerManager.getContainerStatuses(statRequest)
.getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
System.out.println("\nStatus : [" + status + "]\n");
- map.put(status.getState(), status);
- if (map.containsKey(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) &&
- map.containsKey(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) &&
- map.containsKey(
- org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE)) {
+ map.put(status.getContainerSubState(), status);
+ if (map.containsKey(ContainerSubState.RUNNING) &&
+ map.containsKey(ContainerSubState.SCHEDULED) &&
+ map.containsKey(ContainerSubState.DONE)) {
break;
}
Thread.sleep(1000);
}
}
Assert.assertEquals(createContainerId(0),
- map.get(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING)
- .getContainerId());
+ map.get(ContainerSubState.RUNNING).getContainerId());
Assert.assertEquals(createContainerId(1),
- map.get(org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE)
- .getContainerId());
+ map.get(ContainerSubState.DONE).getContainerId());
Assert.assertEquals(createContainerId(2),
- map.get(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED)
- .getContainerId());
+ map.get(ContainerSubState.SCHEDULED).getContainerId());
}
/**
@@ -1186,13 +1170,11 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
if (status.getContainerId().equals(createContainerId(0))) {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
- status.getState());
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
} else {
- Assert.assertEquals(
- org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED,
- status.getState());
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index c547128..d0bfecf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -856,8 +856,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
containers = startEvent.getNMContainerStatuses();
if (containers != null && !containers.isEmpty()) {
for (NMContainerStatus container : containers) {
- if (container.getContainerState() == ContainerState.RUNNING ||
- container.getContainerState() == ContainerState.SCHEDULED) {
+ if (container.getContainerState() == ContainerState.RUNNING) {
rmNode.launchedContainers.add(container.getContainerId());
}
}
@@ -1412,8 +1411,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
// Process running containers
- if (remoteContainer.getState() == ContainerState.RUNNING ||
- remoteContainer.getState() == ContainerState.SCHEDULED) {
+ if (remoteContainer.getState() == ContainerState.RUNNING) {
++numRemoteRunningContainers;
if (!launchedContainers.contains(containerId)) {
// Just launched container. RM knows about it the first time.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d81ae5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 41078d0..2bd745b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -2085,7 +2085,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
ContainerId c2 = ContainerId.newContainerId(appAttemptId, 2);
ContainerId c3 = ContainerId.newContainerId(appAttemptId, 3);
NMContainerStatus queuedOpp =
- NMContainerStatus.newInstance(c1, 1, ContainerState.SCHEDULED,
+ NMContainerStatus.newInstance(c1, 1, ContainerState.RUNNING,
Resource.newInstance(1024, 1), "Dummy Queued OC",
ContainerExitStatus.INVALID, Priority.newInstance(5), 1234, "",
ExecutionType.OPPORTUNISTIC);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-7250. Update Shared cache
client api to use URLs.
Posted by kk...@apache.org.
YARN-7250. Update Shared cache client api to use URLs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c114da5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c114da5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c114da5e
Branch: refs/heads/YARN-6592
Commit: c114da5e64d14b1d9e614081c4171ea0391cb1aa
Parents: 6f789fe
Author: Chris Trezzo <ct...@apache.org>
Authored: Thu Sep 28 15:28:06 2017 -0700
Committer: Chris Trezzo <ct...@apache.org>
Committed: Thu Sep 28 15:28:06 2017 -0700
----------------------------------------------------------------------
.../yarn/client/api/SharedCacheClient.java | 22 ++++--------
.../client/api/impl/SharedCacheClientImpl.java | 36 +++++---------------
.../api/impl/TestSharedCacheClientImpl.java | 31 +++++------------
3 files changed, 23 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
index 60c1bd98..a9c1a07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.impl.SharedCacheClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -58,34 +59,25 @@ public abstract class SharedCacheClient extends AbstractService {
*
* <p>
* The <code>SharedCacheManager</code> responds with whether or not the
- * resource exists in the cache. If the resource exists, a <code>Path</code>
- * to the resource in the shared cache is returned. If the resource does not
+ * resource exists in the cache. If the resource exists, a <code>URL</code> to
+ * the resource in the shared cache is returned. If the resource does not
* exist, null is returned instead.
* </p>
*
* <p>
- * Once a path has been returned for a resource, that path is safe to use for
+ * Once a URL has been returned for a resource, that URL is safe to use for
* the lifetime of the application that corresponds to the provided
* ApplicationId.
* </p>
*
- * <p>
- * Additionally, a name for the resource should be specified. A fragment will
- * be added to the path with the desired name if the desired name is different
- * than the name of the provided path from the shared cache. This ensures that
- * if the returned path is used to create a LocalResource, then the symlink
- * created during YARN localization will match the name specified.
- * </p>
- *
* @param applicationId ApplicationId of the application using the resource
* @param resourceKey the key (i.e. checksum) that identifies the resource
- * @param resourceName the desired name of the resource
- * @return Path to the resource, or null if it does not exist
+ * @return URL to the resource, or null if it does not exist
*/
@Public
@Unstable
- public abstract Path use(ApplicationId applicationId, String resourceKey,
- String resourceName) throws YarnException;
+ public abstract URL use(ApplicationId applicationId, String resourceKey)
+ throws YarnException;
/**
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
index b910c28..3191d36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -38,6 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequ
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.client.api.SharedCacheClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -113,8 +112,8 @@ public class SharedCacheClientImpl extends SharedCacheClient {
}
@Override
- public Path use(ApplicationId applicationId, String resourceKey,
- String resourceName) throws YarnException {
+ public URL use(ApplicationId applicationId, String resourceKey)
+ throws YarnException {
Path resourcePath = null;
UseSharedCacheResourceRequest request = Records.newRecord(
UseSharedCacheResourceRequest.class);
@@ -132,31 +131,12 @@ public class SharedCacheClientImpl extends SharedCacheClient {
throw new YarnException(e);
}
if (resourcePath != null) {
- if (resourcePath.getName().equals(resourceName)) {
- // The preferred name is the same as the name of the item in the cache,
- // so we skip generating the fragment to save space in the MRconfig.
- return resourcePath;
- } else {
- // We are using the shared cache, and a preferred name has been
- // specified that is different than the name of the resource in the
- // shared cache. We need to set the fragment portion of the URI to
- // preserve the desired name.
- URI pathURI = resourcePath.toUri();
- try {
- // We assume that there is no existing fragment in the URI since the
- // shared cache manager does not use fragments.
- pathURI =
- new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(),
- resourceName);
- resourcePath = new Path(pathURI);
- } catch (URISyntaxException e) {
- throw new YarnException(
- "Could not create a new URI due to syntax errors: "
- + pathURI.toString(), e);
- }
- }
+ URL pathURL = URL.fromPath(resourcePath);
+ return pathURL;
+ } else {
+ // The resource was not in the cache.
+ return null;
}
- return resourcePath;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c114da5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
index 16b32db..6a8edd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java
@@ -27,7 +27,6 @@ import static org.mockito.Mockito.when;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,6 +39,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.UseSharedCacheResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
import org.junit.AfterClass;
@@ -114,36 +114,21 @@ public class TestSharedCacheClientImpl {
response.setPath(null);
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
- Path newPath = client.use(mock(ApplicationId.class), "key", null);
- assertNull("The path is not null!", newPath);
+ URL newURL = client.use(mock(ApplicationId.class), "key");
+ assertNull("The path is not null!", newURL);
}
@Test
- public void testUseWithResourceName() throws Exception {
+ public void testUseCacheHit() throws Exception {
Path file = new Path("viewfs://test/path");
- URI useUri = new URI("viewfs://test/path#linkName");
- Path usePath = new Path(useUri);
+ URL useUrl = URL.fromPath(new Path("viewfs://test/path"));
UseSharedCacheResourceResponse response =
new UseSharedCacheResourceResponsePBImpl();
response.setPath(file.toString());
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
response);
- Path newPath = client.use(mock(ApplicationId.class), "key", "linkName");
- assertEquals("The paths are not equal!", usePath, newPath);
- }
-
- @Test
- public void testUseWithSameResourceName() throws Exception {
- Path file = new Path("viewfs://test/path");
- URI useUri = new URI("viewfs://test/path");
- Path usePath = new Path(useUri);
- UseSharedCacheResourceResponse response =
- new UseSharedCacheResourceResponsePBImpl();
- response.setPath(file.toString());
- when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn(
- response);
- Path newPath = client.use(mock(ApplicationId.class), "key", "path");
- assertEquals("The paths are not equal!", usePath, newPath);
+ URL newURL = client.use(mock(ApplicationId.class), "key");
+ assertEquals("The paths are not equal!", useUrl, newURL);
}
@Test(expected = YarnException.class)
@@ -151,7 +136,7 @@ public class TestSharedCacheClientImpl {
String message = "Mock IOExcepiton!";
when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow(
new IOException(message));
- client.use(mock(ApplicationId.class), "key", null);
+ client.use(mock(ApplicationId.class), "key");
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDFS-11968. ViewFS:
StoragePolicies commands fail with HDFS federation. Contributed by Mukul
Kumar Singh.
Posted by kk...@apache.org.
HDFS-11968. ViewFS: StoragePolicies commands fail with HDFS federation. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9130511
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9130511
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9130511
Branch: refs/heads/YARN-6592
Commit: b91305119b434d23b99ae7e755aea6639f48b6ab
Parents: 4d5dd75
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Oct 3 11:23:40 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Oct 3 11:23:40 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/viewfs/InodeTree.java | 3 +-
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 61 +++++++++------
.../hdfs/tools/TestStoragePolicyCommands.java | 11 +--
.../tools/TestViewFSStoragePolicyCommands.java | 80 ++++++++++++++++++++
.../tools/TestWebHDFSStoragePolicyCommands.java | 42 ++++++++++
5 files changed, 168 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
index 665c9c9..7274acd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
@@ -407,7 +407,8 @@ abstract class InodeTree<T> {
for (int j = 1; j <= i; ++j) {
failedAt.append('/').append(path[j]);
}
- throw (new FileNotFoundException(failedAt.toString()));
+ throw (new FileNotFoundException("File/Directory does not exist: "
+ + failedAt.toString()));
}
if (nextInode instanceof INodeLink) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index 9c7d048..d5e5b4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.hdfs.tools;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -29,6 +31,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -100,11 +103,12 @@ public class StoragePolicyAdmin extends Configured implements Tool {
@Override
public int run(Configuration conf, List<String> args) throws IOException {
- final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- Collection<BlockStoragePolicy> policies = dfs.getAllStoragePolicies();
+ Collection<? extends BlockStoragePolicySpi> policies =
+ fs.getAllStoragePolicies();
System.out.println("Block Storage Policies:");
- for (BlockStoragePolicy policy : policies) {
+ for (BlockStoragePolicySpi policy : policies) {
if (policy != null) {
System.out.println("\t" + policy);
}
@@ -149,32 +153,43 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
Path p = new Path(path);
- final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- HdfsFileStatus status = dfs.getClient().getFileInfo(
- Path.getPathWithoutSchemeAndAuthority(p).toString());
- if (status == null) {
+ FileStatus status;
+ try {
+ status = fs.getFileStatus(p);
+ } catch (FileNotFoundException e) {
System.err.println("File/Directory does not exist: " + path);
return 2;
}
- byte storagePolicyId = status.getStoragePolicy();
- if (storagePolicyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
- System.out.println("The storage policy of " + path + " is unspecified");
- return 0;
- }
- Collection<BlockStoragePolicy> policies = dfs.getAllStoragePolicies();
- for (BlockStoragePolicy policy : policies) {
- if (policy.getId() == storagePolicyId) {
- System.out.println("The storage policy of " + path + ":\n" + policy);
+
+ if (status instanceof HdfsFileStatus) {
+ byte storagePolicyId = ((HdfsFileStatus)status).getStoragePolicy();
+ if (storagePolicyId ==
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
+ System.out.println("The storage policy of " + path
+ + " is unspecified");
return 0;
}
+ Collection<? extends BlockStoragePolicySpi> policies =
+ fs.getAllStoragePolicies();
+ for (BlockStoragePolicySpi policy : policies) {
+ if (policy instanceof BlockStoragePolicy) {
+ if (((BlockStoragePolicy)policy).getId() == storagePolicyId) {
+ System.out.println("The storage policy of " + path
+ + ":\n" + policy);
+ return 0;
+ }
+ }
+ }
}
+ System.err.println(getName() + " is not supported for filesystem "
+ + fs.getScheme() + " on path " + path);
+ return 2;
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
return 2;
}
- System.err.println("Cannot identify the storage policy for " + path);
- return 2;
}
}
@@ -218,9 +233,9 @@ public class StoragePolicyAdmin extends Configured implements Tool {
return 1;
}
Path p = new Path(path);
- final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- dfs.setStoragePolicy(p, policyName);
+ fs.setStoragePolicy(p, policyName);
System.out.println("Set storage policy " + policyName + " on " + path);
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
@@ -264,9 +279,9 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
Path p = new Path(path);
- final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- dfs.unsetStoragePolicy(p);
+ fs.unsetStoragePolicy(p);
System.out.println("Unset storage policy from " + path);
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
index 149dabb..f31c739 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
@@ -18,11 +18,12 @@
package org.apache.hadoop.hdfs.tools;
import java.io.IOException;
+import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -38,12 +39,12 @@ public class TestStoragePolicyCommands {
private static final short REPL = 1;
private static final int SIZE = 128;
- private static Configuration conf;
- private static MiniDFSCluster cluster;
- private static DistributedFileSystem fs;
+ protected static Configuration conf;
+ protected static MiniDFSCluster cluster;
+ protected static FileSystem fs;
@Before
- public void clusterSetUp() throws IOException {
+ public void clusterSetUp() throws IOException, URISyntaxException {
conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL).build();
cluster.waitActive();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java
new file mode 100644
index 0000000..b3bb3c4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FsConstants;
+
+import org.apache.hadoop.fs.viewfs.ConfigUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Test StoragePolicyAdmin commands with ViewFileSystem.
+ */
+public class TestViewFSStoragePolicyCommands extends TestStoragePolicyCommands {
+
+ @Before
+ public void clusterSetUp() throws IOException {
+ conf = new HdfsConfiguration();
+ String clusterName = "cluster";
+ cluster =
+ new MiniDFSCluster.Builder(conf).nnTopology(
+ MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem hdfs1 = cluster.getFileSystem(0);
+ DistributedFileSystem hdfs2 = cluster.getFileSystem(1);
+
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ FsConstants.VIEWFS_SCHEME +"://" + clusterName);
+
+ Path base1 = new Path("/user1");
+ Path base2 = new Path("/user2");
+ hdfs1.delete(base1, true);
+ hdfs2.delete(base2, true);
+ hdfs1.mkdirs(base1);
+ hdfs2.mkdirs(base2);
+ ConfigUtil.addLink(conf, clusterName, "/foo",
+ hdfs1.makeQualified(base1).toUri());
+ ConfigUtil.addLink(conf, clusterName, "/hdfs2",
+ hdfs2.makeQualified(base2).toUri());
+ fs = FileSystem.get(conf);
+ }
+
+ /**
+ * Storage policy operation on the viewfs root should fail.
+ */
+ @Test
+ public void testStoragePolicyRoot() throws Exception {
+ final StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /", 2,
+ "is not supported for filesystem viewfs on path /");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java
new file mode 100644
index 0000000..f10205c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * Test StoragePolicyAdmin commands with WebHDFS.
+ */
+public class TestWebHDFSStoragePolicyCommands
+ extends TestStoragePolicyCommands {
+
+ @Before
+ public void clusterSetUp() throws IOException, URISyntaxException {
+ super.clusterSetUp();
+ fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsConstants.WEBHDFS_SCHEME);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ fs.getUri().toString());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HADOOP-14909. Fix the word of
erasure encoding in the top page. Contributed by Takanobu Asanuma.
Posted by kk...@apache.org.
HADOOP-14909. Fix the word of erasure encoding in the top page. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/373d0a51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/373d0a51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/373d0a51
Branch: refs/heads/YARN-6592
Commit: 373d0a51955cabff77e934a28ba2de308207374a
Parents: 8aca46e
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Sep 29 13:34:36 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Sep 29 13:34:36 2017 -0700
----------------------------------------------------------------------
hadoop-project/src/site/markdown/index.md.vm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/373d0a51/hadoop-project/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm
index d9443d6..1526f59 100644
--- a/hadoop-project/src/site/markdown/index.md.vm
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -34,7 +34,7 @@ Minimum required Java version increased from Java 7 to Java 8
All Hadoop JARs are now compiled targeting a runtime version of Java 8.
Users still using Java 7 or below must upgrade to Java 8.
-Support for erasure encoding in HDFS
+Support for erasure coding in HDFS
------------------
Erasure coding is a method for durably storing data with significant space
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-12453.
TestDataNodeHotSwapVolumes fails in trunk Jenkins runs. (Lei (Eddy) Xu)
Posted by kk...@apache.org.
HDFS-12453. TestDataNodeHotSwapVolumes fails in trunk Jenkins runs. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f40dbc17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f40dbc17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f40dbc17
Branch: refs/heads/YARN-6592
Commit: f40dbc170e7f63b947d99b7945e42a14a588bea6
Parents: 7f6118f
Author: Lei Xu <le...@apache.org>
Authored: Fri Sep 29 10:46:17 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Fri Sep 29 10:46:17 2017 -0700
----------------------------------------------------------------------
.../datanode/TestDataNodeHotSwapVolumes.java | 129 ++++++++++++-------
1 file changed, 83 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f40dbc17/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 5d4ac1e..df5e297 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.BlockMissingException;
+import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -36,6 +37,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -46,6 +49,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.After;
@@ -68,6 +72,7 @@ import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -83,6 +88,7 @@ import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -775,12 +781,11 @@ public class TestDataNodeHotSwapVolumes {
private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
throws IOException, ReconfigurationException, TimeoutException,
InterruptedException, BrokenBarrierException {
- // Starts DFS cluster with 3 DataNodes to form a pipeline.
- startDFSCluster(1, 3);
+ startDFSCluster(1, 4);
final short REPLICATION = 3;
- final DataNode dn = cluster.getDataNodes().get(dataNodeIdx);
- final FileSystem fs = cluster.getFileSystem();
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ final DFSClient client = fs.getClient();
final Path testFile = new Path("/test");
FSDataOutputStream out = fs.create(testFile, REPLICATION);
@@ -790,54 +795,93 @@ public class TestDataNodeHotSwapVolumes {
out.write(writeBuf);
out.hflush();
- // Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
- // BlockReceiver releases volume reference before finalizeBlock(), the blocks
- // on the volume will be removed, and finalizeBlock() throws IOE.
- final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
- dn.data = Mockito.spy(data);
- doAnswer(new Answer<Object>() {
- public Object answer(InvocationOnMock invocation)
- throws IOException, InterruptedException {
+ BlockLocation[] blocks = fs.getFileBlockLocations(testFile, 0, BLOCK_SIZE);
+ String[] dataNodeNames = blocks[0].getNames();
+ String dataNodeName = dataNodeNames[dataNodeIdx];
+ int xferPort = Integer.parseInt(dataNodeName.split(":")[1]);
+ DataNode dn = null;
+ for (DataNode dataNode : cluster.getDataNodes()) {
+ if (dataNode.getXferPort() == xferPort) {
+ dn = dataNode;
+ break;
+ }
+ }
+ assertNotNull(dn);
+
+ final CyclicBarrier barrier = new CyclicBarrier(4);
+ final AtomicBoolean done = new AtomicBoolean(false);
+ DataNodeFaultInjector newInjector = new DataNodeFaultInjector() {
+ public void logDelaySendingAckToUpstream(
+ final String upstreamAddr, final long delayMs) throws IOException {
+ try {
+ // Make all streams which hold the volume references to wait the
+ // reconfiguration thread to start.
+ // It should only block IO during the period of reconfiguration
+ // task running.
+ if (!done.get()) {
+ barrier.await();
+ // Add delays to allow the reconfiguration thread starts before
+ // IO finish.
Thread.sleep(1000);
- // Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
- // the block is not removed, since the volume reference should not
- // be released at this point.
- data.finalizeBlock((ExtendedBlock) invocation.getArguments()[0],
- (boolean) invocation.getArguments()[1]);
- return null;
}
- }).when(dn.data).finalizeBlock(any(ExtendedBlock.class),
- Mockito.anyBoolean());
-
- final CyclicBarrier barrier = new CyclicBarrier(2);
+ } catch (InterruptedException | BrokenBarrierException e) {
+ throw new IOException(e);
+ }
+ }
+ };
+ DataNodeFaultInjector oldInjector = DataNodeFaultInjector.get();
- List<String> oldDirs = getDataDirs(dn);
- final String newDirs = oldDirs.get(1); // Remove the first volume.
- final List<Exception> exceptions = new ArrayList<>();
- Thread reconfigThread = new Thread() {
- public void run() {
+ try {
+ DataNodeFaultInjector.set(newInjector);
+
+ List<String> oldDirs = getDataDirs(dn);
+ LocatedBlocks lbs = client.getLocatedBlocks("/test", 0);
+ LocatedBlock block = lbs.get(0);
+ FsVolumeImpl volume =
+ (FsVolumeImpl) dn.getFSDataset().getVolume(block.getBlock());
+ final String newDirs = oldDirs.stream()
+ .filter((d) -> !d.contains(volume.getStorageLocation().toString()))
+ .collect(Collectors.joining(","));
+ final List<IOException> exceptions = new ArrayList<>();
+ final DataNode dataNode = dn;
+ final CyclicBarrier reconfigBarrier = new CyclicBarrier(2);
+
+ Thread reconfigThread = new Thread(() -> {
try {
+ reconfigBarrier.await();
+
+ // Wake up writing threads on the pipeline to finish the block.
barrier.await();
+
assertThat(
"DN did not update its own config",
- dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newDirs),
- is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
+ dataNode.reconfigurePropertyImpl(
+ DFS_DATANODE_DATA_DIR_KEY, newDirs),
+ is(dataNode.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
+ done.set(true);
} catch (ReconfigurationException |
InterruptedException |
BrokenBarrierException e) {
- exceptions.add(e);
+ exceptions.add(new IOException(e));
}
- }
- };
- reconfigThread.start();
+ });
+ reconfigThread.start();
- barrier.await();
- rb.nextBytes(writeBuf);
- out.write(writeBuf);
- out.hflush();
- out.close();
+ // Write more data to make sure the stream threads wait on the barrier.
+ rb.nextBytes(writeBuf);
+ out.write(writeBuf);
+ reconfigBarrier.await();
+ out.hflush();
+ out.close();
+
+ reconfigThread.join();
- reconfigThread.join();
+ if (!exceptions.isEmpty()) {
+ throw MultipleIOException.createIOException(exceptions);
+ }
+ } finally {
+ DataNodeFaultInjector.set(oldInjector);
+ }
// Verify if the data directory reconfigure was successful
FsDatasetSpi<? extends FsVolumeSpi> fsDatasetSpi = dn.getFSDataset();
@@ -851,19 +895,12 @@ public class TestDataNodeHotSwapVolumes {
1, fsVolumeReferences.size());
}
- // Add a new DataNode to help with the pipeline recover.
- cluster.startDataNodes(conf, 1, true, null, null, null);
-
// Verify the file has sufficient replications.
DFSTestUtil.waitReplication(fs, testFile, REPLICATION);
// Read the content back
byte[] content = DFSTestUtil.readFileBuffer(fs, testFile);
assertEquals(BLOCK_SIZE, content.length);
- if (!exceptions.isEmpty()) {
- throw new IOException(exceptions.get(0).getCause());
- }
-
// Write more files to make sure that the DataNode that has removed volume
// is still alive to receive data.
for (int i = 0; i < 10; i++) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-7257. AggregatedLogsBlock
reports a bad 'end' value as a bad 'start' value. Contributed by Jason Lowe
Posted by kk...@apache.org.
YARN-7257. AggregatedLogsBlock reports a bad 'end' value as a bad
'start' value. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28c4957f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28c4957f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28c4957f
Branch: refs/heads/YARN-6592
Commit: 28c4957fccebe2d7e63ec9fe9af58313b4f21d4f
Parents: c87db8d
Author: Xuan <xg...@apache.org>
Authored: Wed Sep 27 15:46:18 2017 -0700
Committer: Xuan <xg...@apache.org>
Committed: Wed Sep 27 15:46:18 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28c4957f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
index f6fea74..ef6876a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
@@ -67,7 +67,7 @@ public class AggregatedLogsBlock extends HtmlBlock {
LogAggregationWebUtils.getLogEndIndex(
html, $("end"));
} catch (NumberFormatException ne) {
- html.h1().__("Invalid log start value: " + $("end")).__();
+ html.h1().__("Invalid log end value: " + $("end")).__();
isValid = false;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-6916. Moving logging APIs
over to slf4j in hadoop-yarn-server-common. Contributed by Bibin A Chundatt
and Akira Ajisaka.
Posted by kk...@apache.org.
YARN-6916. Moving logging APIs over to slf4j in hadoop-yarn-server-common. Contributed by Bibin A Chundatt and Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a877737
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a877737
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a877737
Branch: refs/heads/YARN-6592
Commit: 4a877737182808fe3392a116762ba59973a477a6
Parents: 107c177
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 4 06:06:36 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 4 06:06:39 2017 +0900
----------------------------------------------------------------------
.../hadoop/yarn/server/api/ServerRMProxy.java | 7 +++---
.../OpportunisticContainerAllocator.java | 26 +++++++++-----------
.../BaseContainerTokenSecretManager.java | 24 ++++++++----------
.../security/BaseNMTokenSecretManager.java | 26 ++++++++------------
.../server/sharedcache/SharedCacheUtil.java | 12 ++++-----
.../yarn/server/webapp/AppAttemptBlock.java | 7 +++---
.../hadoop/yarn/server/webapp/AppBlock.java | 6 ++---
.../hadoop/yarn/server/webapp/AppsBlock.java | 6 ++---
.../yarn/server/webapp/ContainerBlock.java | 7 +++---
.../server/webapp/ErrorsAndWarningsBlock.java | 14 +++--------
10 files changed, 60 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
index edec89f..cd92415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
@@ -21,17 +21,18 @@ package org.apache.hadoop.yarn.server.api;
import java.io.IOException;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.client.RMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ServerRMProxy<T> extends RMProxy<T> {
- private static final Log LOG = LogFactory.getLog(ServerRMProxy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServerRMProxy.class);
private ServerRMProxy() {
super();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
index 6fd5228..782dc02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.scheduler;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -42,6 +40,8 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress;
import java.util.ArrayList;
@@ -180,8 +180,8 @@ public class OpportunisticContainerAllocator {
}
}
- private static final Log LOG =
- LogFactory.getLog(OpportunisticContainerAllocator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OpportunisticContainerAllocator.class);
private static final ResourceCalculator RESOURCE_CALCULATOR =
new DominantResourceCalculator();
@@ -255,12 +255,11 @@ public class OpportunisticContainerAllocator {
appContext.getContainerIdGenerator(), appContext.getBlacklist(),
appAttId, appContext.getNodeMap(), userName, containers, anyAsk);
if (!containers.isEmpty()) {
- LOG.info("Opportunistic allocation requested for ["
- + "priority=" + anyAsk.getPriority()
- + ", allocationRequestId=" + anyAsk.getAllocationRequestId()
- + ", num_containers=" + anyAsk.getNumContainers()
- + ", capability=" + anyAsk.getCapability() + "]"
- + " allocated = " + containers.keySet());
+ LOG.info("Opportunistic allocation requested for [priority={}, "
+ + "allocationRequestId={}, num_containers={}, capability={}] "
+ + "allocated = {}", anyAsk.getPriority(),
+ anyAsk.getAllocationRequestId(), anyAsk.getNumContainers(),
+ anyAsk.getCapability(), containers.keySet());
}
}
return containers;
@@ -286,8 +285,7 @@ public class OpportunisticContainerAllocator {
}
if (nodesForScheduling.isEmpty()) {
LOG.warn("No nodes available for allocating opportunistic containers. [" +
- "allNodes=" + allNodes + ", " +
- "blacklist=" + blacklist + "]");
+ "allNodes={}, blacklist={}]", allNodes, blacklist);
return;
}
int numAllocated = 0;
@@ -305,9 +303,9 @@ public class OpportunisticContainerAllocator {
}
cList.add(container);
numAllocated++;
- LOG.info("Allocated [" + container.getId() + "] as opportunistic.");
+ LOG.info("Allocated [{}] as opportunistic.", container.getId());
}
- LOG.info("Allocated " + numAllocated + " opportunistic containers.");
+ LOG.info("Allocated {} opportunistic containers.", numAllocated);
}
private Container buildContainer(long rmIdentifier,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
index e73d07c..a5590fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
@@ -23,14 +23,14 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* SecretManager for ContainerTokens. Extended by both RM and NM and hence is
@@ -40,8 +40,8 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
public class BaseContainerTokenSecretManager extends
SecretManager<ContainerTokenIdentifier> {
- private static Log LOG = LogFactory
- .getLog(BaseContainerTokenSecretManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BaseContainerTokenSecretManager.class);
protected int serialNo = new SecureRandom().nextInt();
@@ -86,11 +86,9 @@ public class BaseContainerTokenSecretManager extends
@Override
public byte[] createPassword(ContainerTokenIdentifier identifier) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Creating password for " + identifier.getContainerID()
- + " for user " + identifier.getUser() + " to be run on NM "
- + identifier.getNmHostAddress());
- }
+ LOG.debug("Creating password for {} for user {} to be run on NM {}",
+ identifier.getContainerID(), identifier.getUser(),
+ identifier.getNmHostAddress());
this.readLock.lock();
try {
return createPassword(identifier.getBytes(),
@@ -114,11 +112,9 @@ public class BaseContainerTokenSecretManager extends
protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
MasterKeyData masterKey)
throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieving password for " + identifier.getContainerID()
- + " for user " + identifier.getUser() + " to be run on NM "
- + identifier.getNmHostAddress());
- }
+ LOG.debug("Retrieving password for {} for user {} to be run on NM {}",
+ identifier.getContainerID(), identifier.getUser(),
+ identifier.getNmHostAddress());
return createPassword(identifier.getBytes(), masterKey.getSecretKey());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
index 759b6f2..3cbd1dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
@@ -24,8 +24,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -35,12 +33,14 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BaseNMTokenSecretManager extends
SecretManager<NMTokenIdentifier> {
- private static Log LOG = LogFactory
- .getLog(BaseNMTokenSecretManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BaseNMTokenSecretManager.class);
protected int serialNo = new SecureRandom().nextInt();
@@ -71,12 +71,9 @@ public class BaseNMTokenSecretManager extends
@Override
protected byte[] createPassword(NMTokenIdentifier identifier) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("creating password for "
- + identifier.getApplicationAttemptId() + " for user "
- + identifier.getApplicationSubmitter() + " to run on NM "
- + identifier.getNodeId());
- }
+ LOG.debug("creating password for {} for user {} to run on NM {}",
+ identifier.getApplicationAttemptId(),
+ identifier.getApplicationSubmitter(), identifier.getNodeId());
readLock.lock();
try {
return createPassword(identifier.getBytes(),
@@ -99,12 +96,9 @@ public class BaseNMTokenSecretManager extends
protected byte[] retrivePasswordInternal(NMTokenIdentifier identifier,
MasterKeyData masterKey) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("creating password for "
- + identifier.getApplicationAttemptId() + " for user "
- + identifier.getApplicationSubmitter() + " to run on NM "
- + identifier.getNodeId());
- }
+ LOG.debug("retriving password for {} for user {} to run on NM {}",
+ identifier.getApplicationAttemptId(),
+ identifier.getApplicationSubmitter(), identifier.getNodeId());
return createPassword(identifier.getBytes(), masterKey.getSecretKey());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
index d3cf379..0e29432 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.yarn.server.sharedcache;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A utility class that contains helper methods for dealing with the internal
@@ -34,7 +34,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
@Unstable
public class SharedCacheUtil {
- private static final Log LOG = LogFactory.getLog(SharedCacheUtil.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SharedCacheUtil.class);
@Private
public static int getCacheDepth(Configuration conf) {
@@ -44,9 +45,8 @@ public class SharedCacheUtil {
if (cacheDepth <= 0) {
LOG.warn("Specified cache depth was less than or equal to zero."
- + " Using default value instead. Default: "
- + YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL
- + ", Specified: " + cacheDepth);
+ + " Using default value instead. Default: {}, Specified: {}",
+ YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL, cacheDepth);
cacheDepth = YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index cad14b6..d663dc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -26,8 +26,6 @@ import java.util.Collection;
import java.util.List;
import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
@@ -46,10 +44,13 @@ import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AppAttemptBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AppAttemptBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected ApplicationAttemptId appAttemptId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index b429b5d..a8810a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -29,8 +29,6 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.UserGroupInformation;
@@ -67,10 +65,12 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AppBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(AppBlock.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AppBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected Configuration conf;
protected ApplicationId appID = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index e3a47de..b2dd4de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -34,8 +34,6 @@ import java.util.List;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.math.LongRange;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
@@ -51,10 +49,12 @@ import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AppsBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(AppsBlock.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AppsBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected EnumSet<YarnApplicationState> reqAppStates;
protected UserGroupInformation callerUGI;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
index b8c0bed..fd48dc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
@@ -23,8 +23,6 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
@@ -38,10 +36,13 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ContainerBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(ContainerBlock.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerBlock.class);
protected ApplicationBaseProtocol appBaseProt;
@Inject
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
index 75f6fcd..4128546 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
@@ -19,11 +19,9 @@
package org.apache.hadoop.yarn.server.webapp;
import com.google.inject.Inject;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericsUtil;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
@@ -60,8 +58,6 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
@Override
protected void render(Block html) {
- Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
-
boolean isAdmin = false;
UserGroupInformation callerUGI = this.getCallerUGI();
@@ -78,7 +74,7 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
return;
}
- if (log instanceof Log4JLogger) {
+ if (GenericsUtil.isLog4jLogger(ErrorsAndWarningsBlock.class)) {
html.__(ErrorMetrics.class);
html.__(WarningMetrics.class);
html.div().button().$onclick("reloadPage()").b("View data for the last ")
@@ -180,8 +176,7 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
cutoffs.add((now - 43200 * 1000) / 1000);
cutoffs.add((now - 84600 * 1000) / 1000);
- Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
- if (log instanceof Log4JLogger) {
+ if (GenericsUtil.isLog4jLogger(ErrorsAndWarningsBlock.class)) {
appender =
Log4jWarningErrorMetricsAppender.findAppender();
}
@@ -193,8 +188,7 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
@Override
protected void render(Block html) {
- Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
- if (log instanceof Log4JLogger) {
+ if (GenericsUtil.isLog4jLogger(ErrorsAndWarningsBlock.class)) {
Hamlet.DIV<Hamlet> div =
html.div().$class("metrics").$style("padding-bottom: 20px");
div.h3(tableHeading).table("#metricsoverview").thead()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-2037. Add work preserving
restart support for Unmanaged AMs. (Botong Huang via Subru).
Posted by kk...@apache.org.
YARN-2037. Add work preserving restart support for Unmanaged AMs. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4d2fd1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4d2fd1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4d2fd1a
Branch: refs/heads/YARN-6592
Commit: d4d2fd1acd2fdddf04f45e67897804eea30d79a1
Parents: 015abcd
Author: Subru Krishnan <su...@apache.org>
Authored: Mon Oct 2 18:14:44 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Mon Oct 2 18:14:44 2017 -0700
----------------------------------------------------------------------
.../yarn/api/ApplicationMasterProtocol.java | 21 ++-
.../records/ApplicationSubmissionContext.java | 17 +-
.../ApplicationMasterService.java | 26 +--
.../resourcemanager/DefaultAMSProcessor.java | 5 +
.../rmapp/attempt/RMAppAttemptImpl.java | 6 +-
.../scheduler/AbstractYarnScheduler.java | 13 +-
.../TestWorkPreservingUnmanagedAM.java | 159 +++++++++++++++++++
7 files changed, 214 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
index 4d78961..eb40fc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
@@ -55,27 +55,32 @@ public interface ApplicationMasterProtocol {
* The interface used by a new <code>ApplicationMaster</code> to register with
* the <code>ResourceManager</code>.
* </p>
- *
+ *
* <p>
* The <code>ApplicationMaster</code> needs to provide details such as RPC
* Port, HTTP tracking url etc. as specified in
* {@link RegisterApplicationMasterRequest}.
* </p>
- *
+ *
* <p>
* The <code>ResourceManager</code> responds with critical details such as
* maximum resource capabilities in the cluster as specified in
* {@link RegisterApplicationMasterResponse}.
* </p>
- *
- * @param request
- * registration request
+ *
+ * <p>
+ * Re-register is only allowed for <code>Unmanaged Application Master</code>
+ * (UAM) HA, with
+ * {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()}
+ * set to true.
+ * </p>
+ *
+ * @param request registration request
* @return registration respose
* @throws YarnException
* @throws IOException
- * @throws InvalidApplicationMasterRequestException
- * The exception is thrown when an ApplicationMaster tries to
- * register more then once.
+ * @throws InvalidApplicationMasterRequestException The exception is thrown
+ * when an ApplicationMaster tries to register more then once.
* @see RegisterApplicationMasterRequest
* @see RegisterApplicationMasterResponse
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 4f1d147..a6bbca7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -395,15 +395,18 @@ public abstract class ApplicationSubmissionContext {
* Set the flag which indicates whether to keep containers across application
* attempts.
* <p>
- * If the flag is true, running containers will not be killed when application
- * attempt fails and these containers will be retrieved by the new application
- * attempt on registration via
+ * For managed AM, if the flag is true, running containers will not be killed
+ * when application attempt fails and these containers will be retrieved by
+ * the new application attempt on registration via
* {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}.
* </p>
- *
- * @param keepContainers
- * the flag which indicates whether to keep containers across
- * application attempts.
+ * <p>
+ * For unmanaged AM, if the flag is true, RM allows re-register and returns
+ * the running containers in the same attempt back to the UAM for HA.
+ * </p>
+ *
+ * @param keepContainers the flag which indicates whether to keep containers
+ * across application attempts.
*/
@Public
@Stable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 479aa43..90c42be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterReque
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
@@ -65,8 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.security
- .AMRMTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.AMRMClientUtils;
@@ -213,14 +213,20 @@ public class ApplicationMasterService extends AbstractService implements
synchronized (lock) {
AllocateResponse lastResponse = lock.getAllocateResponse();
if (hasApplicationMasterRegistered(applicationAttemptId)) {
- String message = AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE + appID;
- LOG.warn(message);
- RMAuditLogger.logFailure(
- this.rmContext.getRMApps()
- .get(appID).getUser(),
- AuditConstants.REGISTER_AM, "", "ApplicationMasterService", message,
- appID, applicationAttemptId);
- throw new InvalidApplicationMasterRequestException(message);
+ // allow UAM re-register if work preservation is enabled
+ ApplicationSubmissionContext appContext =
+ rmContext.getRMApps().get(appID).getApplicationSubmissionContext();
+ if (!(appContext.getUnmanagedAM()
+ && appContext.getKeepContainersAcrossApplicationAttempts())) {
+ String message =
+ AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE + appID;
+ LOG.warn(message);
+ RMAuditLogger.logFailure(
+ this.rmContext.getRMApps().get(appID).getUser(),
+ AuditConstants.REGISTER_AM, "", "ApplicationMasterService",
+ message, appID, applicationAttemptId);
+ throw new InvalidApplicationMasterRequestException(message);
+ }
}
this.amLivelinessMonitor.receivedPing(applicationAttemptId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 5632efe..3eef270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -148,6 +148,11 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
.getTransferredContainers(applicationAttemptId);
if (!transferredContainers.isEmpty()) {
response.setContainersFromPreviousAttempts(transferredContainers);
+ // Clear the node set remembered by the secret manager. Necessary
+ // for UAM restart because we use the same attemptId.
+ rmContext.getNMTokenSecretManager()
+ .clearNodeSetForAttempt(applicationAttemptId);
+
List<NMToken> nmTokens = new ArrayList<NMToken>();
for (Container container : transferredContainers) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index fa29e00..1b4d613 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -363,7 +363,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// Transitions from RUNNING State
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.RUNNING,
- RMAppAttemptEventType.LAUNCHED)
+ EnumSet.of(
+ RMAppAttemptEventType.LAUNCHED,
+ // Valid only for UAM restart
+ RMAppAttemptEventType.REGISTERED))
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.FINAL_SAVING,
RMAppAttemptEventType.UNREGISTERED, new AMUnregisteredTransition())
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.RUNNING,
@@ -1236,7 +1239,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
if (appAttempt.submissionContext
.getKeepContainersAcrossApplicationAttempts()
- && !appAttempt.submissionContext.getUnmanagedAM()
&& rmApp.getCurrentAppAttempt() != appAttempt) {
appAttempt.transferStateFromAttempt(rmApp.getCurrentAppAttempt());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 4896ab0..9d6037e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -243,17 +243,18 @@ public abstract class AbstractYarnScheduler
ApplicationId appId = currentAttempt.getApplicationId();
SchedulerApplication<T> app = applications.get(appId);
List<Container> containerList = new ArrayList<Container>();
- RMApp appImpl = this.rmContext.getRMApps().get(appId);
- if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
- return containerList;
- }
if (app == null) {
return containerList;
}
Collection<RMContainer> liveContainers =
app.getCurrentAppAttempt().getLiveContainers();
- ContainerId amContainerId = rmContext.getRMApps().get(appId)
- .getCurrentAppAttempt().getMasterContainer().getId();
+ ContainerId amContainerId = null;
+ // For UAM, amContainer would be null
+ if (rmContext.getRMApps().get(appId).getCurrentAppAttempt()
+ .getMasterContainer() != null) {
+ amContainerId = rmContext.getRMApps().get(appId).getCurrentAppAttempt()
+ .getMasterContainer().getId();
+ }
for (RMContainer rmContainer : liveContainers) {
if (!rmContainer.getContainerId().equals(amContainerId)) {
containerList.add(rmContainer.getContainer());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4d2fd1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java
new file mode 100644
index 0000000..80f7319
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingUnmanagedAM.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test UAM handling in RM.
+ */
+public class TestWorkPreservingUnmanagedAM
+ extends ParameterizedSchedulerTestBase {
+
+ private YarnConfiguration conf;
+
+ public TestWorkPreservingUnmanagedAM(SchedulerType type) throws IOException {
+ super(type);
+ }
+
+ @Before
+ public void setup() {
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+ conf = getConf();
+ UserGroupInformation.setConfiguration(conf);
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ }
+
+ /**
+ * Test UAM work preserving restart. When the keepContainersAcrossAttempt flag
+ * is on, we allow UAM to directly register again and move on without getting
+ * the applicationAlreadyRegistered exception.
+ */
+ protected void testUAMRestart(boolean keepContainers) throws Exception {
+ // start RM
+ MockRM rm = new MockRM();
+ rm.start();
+ MockNM nm =
+ new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
+ nm.registerNode();
+
+ // create app and launch the UAM
+ boolean unamanged = true;
+ int maxAttempts = 1;
+ boolean waitForAccepted = true;
+ RMApp app = rm.submitApp(200, "",
+ UserGroupInformation.getCurrentUser().getShortUserName(), null,
+ unamanged, null, maxAttempts, null, null, waitForAccepted,
+ keepContainers);
+
+ MockAM am = MockRM.launchUAM(app, rm, nm);
+
+ // Register for the first time
+ am.registerAppAttempt();
+
+ // Allocate two containers to UAM
+ int numContainers = 3;
+ List<Container> conts = am.allocate("127.0.0.1", 1000, numContainers,
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (conts.size() < numContainers) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(100);
+ }
+
+ // Release one container
+ List<ContainerId> releaseList =
+ Collections.singletonList(conts.get(0).getId());
+ List<ContainerStatus> finishedConts =
+ am.allocate(new ArrayList<ResourceRequest>(), releaseList)
+ .getCompletedContainersStatuses();
+ while (finishedConts.size() < releaseList.size()) {
+ nm.nodeHeartbeat(true);
+ finishedConts
+ .addAll(am
+ .allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>())
+ .getCompletedContainersStatuses());
+ Thread.sleep(100);
+ }
+
+ // Register for the second time
+ RegisterApplicationMasterResponse response = null;
+ try {
+ response = am.registerAppAttempt(false);
+ } catch (InvalidApplicationMasterRequestException e) {
+ Assert.assertEquals(false, keepContainers);
+ return;
+ }
+ Assert.assertEquals("RM should not allow second register"
+ + " for UAM without keep container flag ", true, keepContainers);
+
+ // Expecting the two running containers previously
+ Assert.assertEquals(2, response.getContainersFromPreviousAttempts().size());
+ Assert.assertEquals(1, response.getNMTokensFromPreviousAttempts().size());
+
+ // Allocate one more containers to UAM, just to be safe
+ numContainers = 1;
+ am.allocate("127.0.0.1", 1000, numContainers, new ArrayList<ContainerId>());
+ nm.nodeHeartbeat(true);
+ conts = am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ while (conts.size() < numContainers) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(100);
+ }
+
+ rm.stop();
+ }
+
+ @Test(timeout = 600000)
+ public void testUAMRestartKeepContainers() throws Exception {
+ testUAMRestart(true);
+ }
+
+ @Test(timeout = 600000)
+ public void testUAMRestartNoKeepContainers() throws Exception {
+ testUAMRestart(false);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-6593. [API] Introduce
Placement Constraint object. (Konstantinos Karanasos via wangda)
Posted by kk...@apache.org.
YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda)
Change-Id: Id00edb7185fdf01cce6e40f920cac3585f8cbe9c
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/229680a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/229680a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/229680a0
Branch: refs/heads/YARN-6592
Commit: 229680a064da2d4b0e6bc270c925c2d24981e5db
Parents: 4a87773
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Aug 3 14:03:55 2017 -0700
Committer: Konstantinos Karanasos <kk...@apache.org>
Committed: Tue Oct 3 14:14:53 2017 -0700
----------------------------------------------------------------------
.../yarn/api/resource/PlacementConstraint.java | 567 +++++++++++++++++++
.../yarn/api/resource/PlacementConstraints.java | 286 ++++++++++
.../hadoop/yarn/api/resource/package-info.java | 23 +
.../src/main/proto/yarn_protos.proto | 55 ++
.../api/resource/TestPlacementConstraints.java | 106 ++++
.../PlacementConstraintFromProtoConverter.java | 116 ++++
.../pb/PlacementConstraintToProtoConverter.java | 174 ++++++
.../apache/hadoop/yarn/api/pb/package-info.java | 23 +
.../yarn/api/records/impl/pb/ProtoUtils.java | 27 +
.../PlacementConstraintTransformations.java | 209 +++++++
.../hadoop/yarn/api/resource/package-info.java | 23 +
.../TestPlacementConstraintPBConversion.java | 195 +++++++
.../TestPlacementConstraintTransformations.java | 183 ++++++
13 files changed, 1987 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
new file mode 100644
index 0000000..f0e3982
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -0,0 +1,567 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * {@code PlacementConstraint} represents a placement constraint for a resource
+ * allocation.
+ */
+@Public
+@Unstable
+public class PlacementConstraint {
+
+ /**
+ * The constraint expression tree.
+ */
+ private AbstractConstraint constraintExpr;
+
+ public PlacementConstraint(AbstractConstraint constraintExpr) {
+ this.constraintExpr = constraintExpr;
+ }
+
+ /**
+ * Get the constraint expression of the placement constraint.
+ *
+ * @return the constraint expression
+ */
+ public AbstractConstraint getConstraintExpr() {
+ return constraintExpr;
+ }
+
+ /**
+ * Interface used to enable the elements of the constraint tree to be visited.
+ */
+ @Private
+ public interface Visitable {
+ /**
+ * Visitor pattern.
+ *
+ * @param visitor visitor to be used
+ * @param <T> defines the type that the visitor will use and the return type
+ * of the accept.
+ * @return the result of visiting a given object.
+ */
+ <T> T accept(Visitor<T> visitor);
+
+ }
+
+ /**
+ * Visitor API for a constraint tree.
+ *
+ * @param <T> determines the return type of the visit methods.
+ */
+ @Private
+ public interface Visitor<T> {
+ T visit(SingleConstraint constraint);
+
+ T visit(TargetExpression target);
+
+ T visit(TargetConstraint constraint);
+
+ T visit(CardinalityConstraint constraint);
+
+ T visit(And constraint);
+
+ T visit(Or constraint);
+
+ T visit(DelayedOr constraint);
+
+ T visit(TimedPlacementConstraint constraint);
+ }
+
+ /**
+ * Abstract class that acts as the superclass of all placement constraint
+ * classes.
+ */
+ public abstract static class AbstractConstraint implements Visitable {
+ public PlacementConstraint build() {
+ return new PlacementConstraint(this);
+ }
+ }
+
+ static final String NODE_SCOPE = "node";
+ static final String RACK_SCOPE = "rack";
+
+ /**
+ * Consider a set of nodes N that belongs to the scope specified in the
+ * constraint. If the target expressions are satisfied at least minCardinality
+ * times and at most max-cardinality times in the node set N, then the
+ * constraint is satisfied.
+ *
+ * For example, a constraint of the form {@code {RACK, 2, 10,
+ * allocationTag("zk")}}, requires an allocation to be placed within a rack
+ * that has at least 2 and at most 10 other allocations with tag "zk".
+ */
+ public static class SingleConstraint extends AbstractConstraint {
+ private String scope;
+ private int minCardinality;
+ private int maxCardinality;
+ private Set<TargetExpression> targetExpressions;
+
+ public SingleConstraint(String scope, int minCardinality,
+ int maxCardinality, Set<TargetExpression> targetExpressions) {
+ this.scope = scope;
+ this.minCardinality = minCardinality;
+ this.maxCardinality = maxCardinality;
+ this.targetExpressions = targetExpressions;
+ }
+
+ public SingleConstraint(String scope, int minC, int maxC,
+ TargetExpression... targetExpressions) {
+ this(scope, minC, maxC, new HashSet<>(Arrays.asList(targetExpressions)));
+ }
+
+ /**
+ * Get the scope of the constraint.
+ *
+ * @return the scope of the constraint
+ */
+ public String getScope() {
+ return scope;
+ }
+
+ /**
+ * Get the minimum cardinality of the constraint.
+ *
+ * @return the minimum cardinality of the constraint
+ */
+ public int getMinCardinality() {
+ return minCardinality;
+ }
+
+ /**
+ * Get the maximum cardinality of the constraint.
+ *
+ * @return the maximum cardinality of the constraint
+ */
+ public int getMaxCardinality() {
+ return maxCardinality;
+ }
+
+ /**
+ * Get the target expressions of the constraint.
+ *
+ * @return the set of target expressions
+ */
+ public Set<TargetExpression> getTargetExpressions() {
+ return targetExpressions;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class representing the target expressions that are used in placement
+ * constraints. They might refer to expressions on node attributes, allocation
+ * tags, or be self-targets (referring to the allocation to which the
+ * constraint is attached).
+ */
+ public static class TargetExpression implements Visitable {
+ /**
+ * Enum specifying the type of the target expression.
+ */
+ public enum TargetType {
+ NODE_ATTRIBUTE, ALLOCATION_TAG, SELF
+ }
+
+ private TargetType targetType;
+ private String targetKey;
+ private Set<String> targetValues;
+
+ public TargetExpression(TargetType targetType, String targetKey,
+ Set<String> targetValues) {
+ this.targetType = targetType;
+ this.targetKey = targetKey;
+ this.targetValues = targetValues;
+ }
+
+ public TargetExpression(TargetType targetType) {
+ this(targetType, null, new HashSet<>());
+ }
+
+ public TargetExpression(TargetType targetType, String targetKey,
+ String... targetValues) {
+ this(targetType, targetKey, new HashSet<>(Arrays.asList(targetValues)));
+ }
+
+ /**
+ * Get the type of the target expression.
+ *
+ * @return the type of the target expression
+ */
+ public TargetType getTargetType() {
+ return targetType;
+ }
+
+ /**
+ * Get the key of the target expression.
+ *
+ * @return the key of the target expression
+ */
+ public String getTargetKey() {
+ return targetKey;
+ }
+
+ /**
+ * Get the set of values of the target expression.
+ *
+ * @return the set of values of the target expression
+ */
+ public Set<String> getTargetValues() {
+ return targetValues;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = targetType != null ? targetType.hashCode() : 0;
+ result = 31 * result + (targetKey != null ? targetKey.hashCode() : 0);
+ result =
+ 31 * result + (targetValues != null ? targetValues.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null) {
+ return false;
+ }
+ if (!(o instanceof TargetExpression)) {
+ return false;
+ }
+
+ TargetExpression that = (TargetExpression) o;
+ if (targetType != that.targetType) {
+ return false;
+ }
+ if (targetKey != null ? !targetKey.equals(that.targetKey)
+ : that.targetKey != null) {
+ return false;
+ }
+ return targetValues != null ? targetValues.equals(that.targetValues)
+ : that.targetValues == null;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a target constraint. Such a constraint requires an
+ * allocation to be placed within a scope that satisfies some specified
+ * expressions on node attributes and allocation tags.
+ *
+ * It is a specialized version of the {@link SingleConstraint}, where the
+ * minimum and the maximum cardinalities take specific values based on the
+ * {@link TargetOperator} used.
+ */
+ public static class TargetConstraint extends AbstractConstraint {
+ enum TargetOperator {
+ IN, NOT_IN
+ }
+
+ private TargetOperator op;
+ private String scope;
+ private Set<TargetExpression> targetExpressions;
+
+ public TargetConstraint(TargetOperator op, String scope,
+ Set<TargetExpression> targetExpressions) {
+ this.op = op;
+ this.scope = scope;
+ this.targetExpressions = targetExpressions;
+ }
+
+ /**
+ * Get the target operator of the constraint.
+ *
+ * @return the target operator
+ */
+ public TargetOperator getOp() {
+ return op;
+ }
+
+ /**
+ * Get the scope of the constraint.
+ *
+ * @return the scope of the constraint
+ */
+ public String getScope() {
+ return scope;
+ }
+
+ /**
+ * Get the set of target expressions.
+ *
+ * @return the set of target expressions
+ */
+ public Set<TargetExpression> getTargetExpressions() {
+ return targetExpressions;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a cardinality constraint. Such a constraint the
+ * number of allocations within a given scope to some minimum and maximum
+ * values.
+ *
+ * It is a specialized version of the {@link SingleConstraint}, where the
+ * target is self (i.e., the allocation to which the constraint is attached).
+ */
+ public static class CardinalityConstraint extends AbstractConstraint {
+ private String scope;
+ private int minCardinality;
+ private int maxCardinality;
+
+ public CardinalityConstraint(String scope, int minCardinality,
+ int maxCardinality) {
+ this.scope = scope;
+ this.minCardinality = minCardinality;
+ this.maxCardinality = maxCardinality;
+ }
+
+ /**
+ * Get the scope of the constraint.
+ *
+ * @return the scope of the constraint
+ */
+ public String getScope() {
+ return scope;
+ }
+
+ /**
+ * Get the minimum cardinality of the constraint.
+ *
+ * @return the minimum cardinality of the constraint
+ */
+ public int getMinCardinality() {
+ return minCardinality;
+ }
+
+ /**
+ * Get the maximum cardinality of the constraint.
+ *
+ * @return the maximum cardinality of the constraint
+ */
+ public int getMaxCardinality() {
+ return maxCardinality;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents composite constraints, which comprise other
+ * constraints, forming a constraint tree.
+ *
+ * @param <R> the type of constraints that are used as children of the
+ * specific composite constraint
+ */
+ public abstract static class CompositeConstraint<R extends Visitable>
+ extends AbstractConstraint {
+
+ /**
+ * Get the children of this composite constraint.
+ *
+ * @return the children of the composite constraint
+ */
+ public abstract List<R> getChildren();
+ }
+
+ /**
+ * Class that represents a composite constraint that is a conjunction of other
+ * constraints.
+ */
+ public static class And extends CompositeConstraint<AbstractConstraint> {
+ private List<AbstractConstraint> children;
+
+ public And(List<AbstractConstraint> children) {
+ this.children = children;
+ }
+
+ public And(AbstractConstraint... children) {
+ this(Arrays.asList(children));
+ }
+
+ @Override
+ public List<AbstractConstraint> getChildren() {
+ return children;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a composite constraint that is a disjunction of other
+ * constraints.
+ */
+ public static class Or extends CompositeConstraint<AbstractConstraint> {
+ private List<AbstractConstraint> children;
+
+ public Or(List<AbstractConstraint> children) {
+ this.children = children;
+ }
+
+ public Or(AbstractConstraint... children) {
+ this(Arrays.asList(children));
+ }
+
+ @Override
+ public List<AbstractConstraint> getChildren() {
+ return children;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a composite constraint that comprises a list of timed
+ * placement constraints (see {@link TimedPlacementConstraint}). The scheduler
+ * should try to satisfy first the first timed child constraint within the
+ * specified time window. If this is not possible, it should attempt to
+ * satisfy the second, and so on.
+ */
+ public static class DelayedOr
+ extends CompositeConstraint<TimedPlacementConstraint> {
+ private List<TimedPlacementConstraint> children = new ArrayList<>();
+
+ public DelayedOr(List<TimedPlacementConstraint> children) {
+ this.children = children;
+ }
+
+ public DelayedOr(TimedPlacementConstraint... children) {
+ this(Arrays.asList(children));
+ }
+
+ @Override
+ public List<TimedPlacementConstraint> getChildren() {
+ return children;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Represents a timed placement constraint that has to be satisfied within a
+ * time window.
+ */
+ public static class TimedPlacementConstraint implements Visitable {
+ /**
+ * The unit of scheduling delay.
+ */
+ public enum DelayUnit {
+ MILLISECONDS, OPPORTUNITIES
+ }
+
+ private AbstractConstraint constraint;
+ private long schedulingDelay;
+ private DelayUnit delayUnit;
+
+ public TimedPlacementConstraint(AbstractConstraint constraint,
+ long schedulingDelay, DelayUnit delayUnit) {
+ this.constraint = constraint;
+ this.schedulingDelay = schedulingDelay;
+ this.delayUnit = delayUnit;
+ }
+
+ public TimedPlacementConstraint(AbstractConstraint constraint,
+ long schedulingDelay) {
+ this(constraint, schedulingDelay, DelayUnit.MILLISECONDS);
+ }
+
+ public TimedPlacementConstraint(AbstractConstraint constraint) {
+ this(constraint, Long.MAX_VALUE, DelayUnit.MILLISECONDS);
+ }
+
+ /**
+ * Get the constraint that has to be satisfied within the time window.
+ *
+ * @return the constraint to be satisfied
+ */
+ public AbstractConstraint getConstraint() {
+ return constraint;
+ }
+
+ /**
+ * Sets the constraint that has to be satisfied within the time window.
+ *
+ * @param constraint the constraint to be satisfied
+ */
+ public void setConstraint(AbstractConstraint constraint) {
+ this.constraint = constraint;
+ }
+
+ /**
+ * Get the scheduling delay value that determines the time window within
+ * which the constraint has to be satisfied.
+ *
+ * @return the value of the scheduling delay
+ */
+ public long getSchedulingDelay() {
+ return schedulingDelay;
+ }
+
+ /**
+ * The unit of the scheduling delay.
+ *
+ * @return the unit of the delay
+ */
+ public DelayUnit getDelayUnit() {
+ return delayUnit;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
new file mode 100644
index 0000000..8e84280
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
@@ -0,0 +1,286 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+
+/**
+ * This class contains various static methods for the applications to create
+ * placement constraints (see also {@link PlacementConstraint}).
+ */
+@Public
+@Unstable
+public final class PlacementConstraints {
+
+ // Suppresses default constructor, ensuring non-instantiability.
+ private PlacementConstraints() {
+ }
+
+ // Creation of simple constraints.
+
+ public static final String NODE = PlacementConstraint.NODE_SCOPE;
+ public static final String RACK = PlacementConstraint.RACK_SCOPE;
+
+ /**
+ * Creates a constraint that requires allocations to be placed on nodes that
+ * satisfy all target expressions within the given scope (e.g., node or rack).
+ *
+ * For example, {@code targetIn(RACK, allocationTag("hbase-m"))}, allows
+ * allocations on nodes that belong to a rack that has at least one tag with
+ * value "hbase-m".
+ *
+ * @param scope the scope within which the target expressions should be
+ * satisfied
+ * @param targetExpressions the expressions that need to be satisfied within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint targetIn(String scope,
+ TargetExpression... targetExpressions) {
+ return new SingleConstraint(scope, 1, Integer.MAX_VALUE, targetExpressions);
+ }
+
+ /**
+ * Creates a constraint that requires allocations to be placed on nodes that
+ * belong to a scope (e.g., node or rack) that does not satisfy any of the
+ * target expressions.
+ *
+ * @param scope the scope within which the target expressions should not be
+ * true
+ * @param targetExpressions the expressions that need to not be true within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint targetNotIn(String scope,
+ TargetExpression... targetExpressions) {
+ return new SingleConstraint(scope, 0, 0, targetExpressions);
+ }
+
+ /**
+ * Creates a constraint that restricts the number of allocations within a
+ * given scope (e.g., node or rack).
+ *
+ * For example, {@code cardinality(NODE, 3, 10)}, restricts the number of
+ * allocations per node to be no less than 3 and no more than 10.
+ *
+ * @param scope the scope of the constraint
+ * @param minCardinality determines the minimum number of allocations within
+ * the scope
+ * @param maxCardinality determines the maximum number of allocations within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint cardinality(String scope, int minCardinality,
+ int maxCardinality) {
+ return new SingleConstraint(scope, minCardinality, maxCardinality,
+ PlacementTargets.self());
+ }
+
+ /**
+ * Similar to {@link #cardinality(String, int, int)}, but determines only the
+ * minimum cardinality (the maximum cardinality is unbound).
+ *
+ * @param scope the scope of the constraint
+ * @param minCardinality determines the minimum number of allocations within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint minCardinality(String scope,
+ int minCardinality) {
+ return cardinality(scope, minCardinality, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Similar to {@link #cardinality(String, int, int)}, but determines only the
+ * maximum cardinality (the minimum can be as low as 0).
+ *
+ * @param scope the scope of the constraint
+ * @param maxCardinality determines the maximum number of allocations within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint maxCardinality(String scope,
+ int maxCardinality) {
+ return cardinality(scope, 0, maxCardinality);
+ }
+
+ /**
+ * This constraint generalizes the cardinality and target constraints.
+ *
+ * Consider a set of nodes N that belongs to the scope specified in the
+ * constraint. If the target expressions are satisfied at least minCardinality
+ * times and at most max-cardinality times in the node set N, then the
+ * constraint is satisfied.
+ *
+ * For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))},
+ * requires an allocation to be placed within a rack that has at least 2 and
+ * at most 10 other allocations with tag "zk".
+ *
+ * @param scope the scope of the constraint
+ * @param minCardinality the minimum number of times the target expressions
+ * have to be satisfied with the given scope
+ * @param maxCardinality the maximum number of times the target expressions
+ * have to be satisfied with the given scope
+ * @param targetExpressions the target expressions
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint targetCardinality(String scope,
+ int minCardinality, int maxCardinality,
+ TargetExpression... targetExpressions) {
+ return new SingleConstraint(scope, minCardinality, maxCardinality,
+ targetExpressions);
+ }
+
+ // Creation of target expressions to be used in simple constraints.
+
+ /**
+ * Class with static methods for constructing target expressions to be used in
+ * placement constraints.
+ */
+ public static class PlacementTargets {
+
+ /**
+ * Constructs a target expression on a node attribute. It is satisfied if
+ * the specified node attribute has one of the specified values.
+ *
+ * @param attributeKey the name of the node attribute
+ * @param attributeValues the set of values that the attribute should take
+ * values from
+ * @return the resulting expression on the node attribute
+ */
+ public static TargetExpression nodeAttribute(String attributeKey,
+ String... attributeValues) {
+ return new TargetExpression(TargetType.NODE_ATTRIBUTE, attributeKey,
+ attributeValues);
+ }
+
+ /**
+ * Constructs a target expression on an allocation tag. It is satisfied if
+ * the there are allocations with one of the given tags.
+ *
+ * @param allocationTags the set of tags that the attribute should take
+ * values from
+ * @return the resulting expression on the allocation tags
+ */
+ public static TargetExpression allocationTag(String... allocationTags) {
+ return new TargetExpression(TargetType.ALLOCATION_TAG, null,
+ allocationTags);
+ }
+
+ /**
+ * The default target expression that uses as target the allocation that
+ * specifies the constraint.
+ *
+ * @return the self-target
+ */
+ public static TargetExpression self() {
+ return new TargetExpression(TargetType.SELF);
+ }
+ }
+
+ // Creation of compound constraints.
+
+ /**
+ * A conjunction of constraints.
+ *
+ * @param children the children constraints that should all be satisfied
+ * @return the resulting placement constraint
+ */
+ public static And and(AbstractConstraint... children) {
+ return new And(children);
+ }
+
+ /**
+ * A disjunction of constraints.
+ *
+ * @param children the children constraints, one of which should be satisfied
+ * @return the resulting placement constraint
+ */
+ public static Or or(AbstractConstraint... children) {
+ return new Or(children);
+ }
+
+ /**
+ * Creates a composite constraint that includes a list of timed placement
+ * constraints. The scheduler should try to satisfy first the first timed
+ * child constraint within the specified time window. If this is not possible,
+ * it should attempt to satisfy the second, and so on.
+ *
+ * @param children the timed children constraints
+ * @return the resulting composite constraint
+ */
+ public static DelayedOr delayedOr(TimedPlacementConstraint... children) {
+ return new DelayedOr(children);
+ }
+
+ // Creation of timed constraints to be used in a DELAYED_OR constraint.
+
+ /**
+ * Creates a placement constraint that has to be satisfied within a time
+ * window.
+ *
+ * @param constraint the placement constraint
+ * @param delay the length of the time window within which the constraint has
+ * to be satisfied
+ * @param timeUnit the unit of time of the time window
+ * @return the resulting timed placement constraint
+ */
+ public static TimedPlacementConstraint timedClockConstraint(
+ AbstractConstraint constraint, long delay, TimeUnit timeUnit) {
+ return new TimedPlacementConstraint(constraint, timeUnit.toMillis(delay),
+ TimedPlacementConstraint.DelayUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a placement constraint that has to be satisfied within a number of
+ * placement opportunities (invocations of the scheduler).
+ *
+ * @param constraint the placement constraint
+ * @param delay the number of scheduling opportunities within which the
+ * constraint has to be satisfied
+ * @return the resulting timed placement constraint
+ */
+ public static TimedPlacementConstraint timedOpportunitiesConstraint(
+ AbstractConstraint constraint, long delay) {
+ return new TimedPlacementConstraint(constraint, delay,
+ TimedPlacementConstraint.DelayUnit.OPPORTUNITIES);
+ }
+
+ /**
+ * Creates a {@link PlacementConstraint} given a constraint expression.
+ *
+ * @param constraintExpr the constraint expression
+ * @return the placement constraint
+ */
+ public static PlacementConstraint build(AbstractConstraint constraintExpr) {
+ return constraintExpr.build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
new file mode 100644
index 0000000..660dc02
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * API related to resources.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.api.resource;
+import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index e69c07b..57ebcc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -566,6 +566,61 @@ enum SignalContainerCommandProto {
FORCEFUL_SHUTDOWN = 3;
}
+////////////////////////////////////////////////////////////////////////
+////// Placement constraints ///////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////
+
+message PlacementConstraintProto {
+ optional SimplePlacementConstraintProto simpleConstraint = 1;
+ optional CompositePlacementConstraintProto compositeConstraint = 2;
+}
+
+message SimplePlacementConstraintProto {
+ required string scope = 1;
+ repeated PlacementConstraintTargetProto targetExpressions = 2;
+ optional int32 minCardinality = 3;
+ optional int32 maxCardinality = 4;
+}
+
+message PlacementConstraintTargetProto {
+ enum TargetType {
+ NODE_ATTRIBUTE = 1;
+ ALLOCATION_TAG = 2;
+ SELF = 3;
+ }
+
+ required TargetType targetType = 1;
+ optional string targetKey = 2;
+ repeated string targetValues = 3;
+}
+
+message TimedPlacementConstraintProto {
+ enum DelayUnit {
+ MILLISECONDS = 1;
+ OPPORTUNITIES = 2;
+ }
+
+ required PlacementConstraintProto placementConstraint = 1;
+ required int64 schedulingDelay = 2;
+ optional DelayUnit delayUnit = 3 [ default = MILLISECONDS ];
+}
+
+message CompositePlacementConstraintProto {
+ enum CompositeType {
+ // All children constraints have to be satisfied.
+ AND = 1;
+ // One of the children constraints has to be satisfied.
+ OR = 2;
+ // Attempt to satisfy the first child constraint for delays[0] units (e.g.,
+ // millisec or heartbeats). If this fails, try to satisfy the second child
+ // constraint for delays[1] units and so on.
+ DELAYED_OR = 3;
+ }
+
+ required CompositeType compositeType = 1;
+ repeated PlacementConstraintProto childConstraints = 2;
+ repeated TimedPlacementConstraintProto timedChildConstraints = 3;
+}
////////////////////////////////////////////////////////////////////////
////// From reservation_protocol /////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
new file mode 100644
index 0000000..e25d477
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.and;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute;
+
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test class for the various static methods in
+ * {@link org.apache.hadoop.yarn.api.resource.PlacementConstraints}.
+ */
+public class TestPlacementConstraints {
+
+ @Test
+ public void testNodeAffinityToTag() {
+ AbstractConstraint constraintExpr =
+ targetIn(NODE, allocationTag("hbase-m"));
+
+ SingleConstraint sConstraint = (SingleConstraint) constraintExpr;
+ Assert.assertEquals(NODE, sConstraint.getScope());
+ Assert.assertEquals(1, sConstraint.getMinCardinality());
+ Assert.assertEquals(Integer.MAX_VALUE, sConstraint.getMaxCardinality());
+
+ Assert.assertEquals(1, sConstraint.getTargetExpressions().size());
+ TargetExpression tExpr =
+ sConstraint.getTargetExpressions().iterator().next();
+ Assert.assertNull(tExpr.getTargetKey());
+ Assert.assertEquals(TargetType.ALLOCATION_TAG, tExpr.getTargetType());
+ Assert.assertEquals(1, tExpr.getTargetValues().size());
+ Assert.assertEquals("hbase-m", tExpr.getTargetValues().iterator().next());
+
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+ Assert.assertNotNull(constraint.getConstraintExpr());
+ }
+
+ @Test
+ public void testNodeAntiAffinityToAttribute() {
+ AbstractConstraint constraintExpr =
+ targetNotIn(NODE, nodeAttribute("java", "1.8"));
+
+ SingleConstraint sConstraint = (SingleConstraint) constraintExpr;
+ Assert.assertEquals(NODE, sConstraint.getScope());
+ Assert.assertEquals(0, sConstraint.getMinCardinality());
+ Assert.assertEquals(0, sConstraint.getMaxCardinality());
+
+ Assert.assertEquals(1, sConstraint.getTargetExpressions().size());
+ TargetExpression tExpr =
+ sConstraint.getTargetExpressions().iterator().next();
+ Assert.assertEquals("java", tExpr.getTargetKey());
+ Assert.assertEquals(TargetType.NODE_ATTRIBUTE, tExpr.getTargetType());
+ Assert.assertEquals(1, tExpr.getTargetValues().size());
+ Assert.assertEquals("1.8", tExpr.getTargetValues().iterator().next());
+ }
+
+ @Test
+ public void testAndConstraint() {
+ AbstractConstraint constraintExpr =
+ and(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
+ targetCardinality(RACK, 2, 10, allocationTag("zk")));
+
+ And andExpr = (And) constraintExpr;
+ Assert.assertEquals(3, andExpr.getChildren().size());
+ SingleConstraint sConstr = (SingleConstraint) andExpr.getChildren().get(0);
+ TargetExpression tExpr = sConstr.getTargetExpressions().iterator().next();
+ Assert.assertEquals("spark", tExpr.getTargetValues().iterator().next());
+
+ sConstr = (SingleConstraint) andExpr.getChildren().get(1);
+ Assert.assertEquals(0, sConstr.getMinCardinality());
+ Assert.assertEquals(3, sConstr.getMaxCardinality());
+
+ sConstr = (SingleConstraint) andExpr.getChildren().get(2);
+ Assert.assertEquals(2, sConstr.getMinCardinality());
+ Assert.assertEquals(10, sConstr.getMaxCardinality());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java
new file mode 100644
index 0000000..926b6fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.pb;
+
+import static org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType.AND;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto;
+
+/**
+ * {@code PlacementConstraintFromProtoConverter} generates an
+ * {@link PlacementConstraint.AbstractConstraint} given a
+ * {@link PlacementConstraintProto}.
+ */
+@Private
+public class PlacementConstraintFromProtoConverter {
+
+ private PlacementConstraintProto constraintProto;
+
+ public PlacementConstraintFromProtoConverter(
+ PlacementConstraintProto constraintProto) {
+ this.constraintProto = constraintProto;
+ }
+
+ public PlacementConstraint convert() {
+ return new PlacementConstraint(convert(constraintProto));
+ }
+
+ private AbstractConstraint convert(PlacementConstraintProto proto) {
+ return proto.hasSimpleConstraint() ? convert(proto.getSimpleConstraint())
+ : convert(proto.getCompositeConstraint());
+ }
+
+ private SingleConstraint convert(SimplePlacementConstraintProto proto) {
+ Set<TargetExpression> targets = new HashSet<>();
+ for (PlacementConstraintTargetProto tp : proto.getTargetExpressionsList()) {
+ targets.add(convert(tp));
+ }
+
+ return new SingleConstraint(proto.getScope(), proto.getMinCardinality(),
+ proto.getMaxCardinality(), targets);
+ }
+
+ private TargetExpression convert(PlacementConstraintTargetProto proto) {
+ return new TargetExpression(
+ ProtoUtils.convertFromProtoFormat(proto.getTargetType()),
+ proto.hasTargetKey() ? proto.getTargetKey() : null,
+ new HashSet<>(proto.getTargetValuesList()));
+ }
+
+ private AbstractConstraint convert(CompositePlacementConstraintProto proto) {
+ switch (proto.getCompositeType()) {
+ case AND:
+ case OR:
+ List<AbstractConstraint> children = new ArrayList<>();
+ for (PlacementConstraintProto cp : proto.getChildConstraintsList()) {
+ children.add(convert(cp));
+ }
+ return (proto.getCompositeType() == AND) ? new And(children)
+ : new Or(children);
+ case DELAYED_OR:
+ List<TimedPlacementConstraint> tChildren = new ArrayList<>();
+ for (TimedPlacementConstraintProto cp : proto
+ .getTimedChildConstraintsList()) {
+ tChildren.add(convert(cp));
+ }
+ return new DelayedOr(tChildren);
+ default:
+ throw new YarnRuntimeException(
+ "Encountered unexpected type of composite constraint.");
+ }
+ }
+
+ private TimedPlacementConstraint convert(
+ TimedPlacementConstraintProto proto) {
+ AbstractConstraint pConstraint = convert(proto.getPlacementConstraint());
+
+ return new TimedPlacementConstraint(pConstraint, proto.getSchedulingDelay(),
+ ProtoUtils.convertFromProtoFormat(proto.getDelayUnit()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
new file mode 100644
index 0000000..7816e18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto;
+
+import com.google.protobuf.GeneratedMessage;
+
+/**
+ * {@code PlacementConstraintToProtoConverter} generates a
+ * {@link PlacementConstraintProto} given a
+ * {@link PlacementConstraint.AbstractConstraint}.
+ */
+@Private
+public class PlacementConstraintToProtoConverter
+ implements PlacementConstraint.Visitor<GeneratedMessage> {
+
+ private PlacementConstraint placementConstraint;
+
+ public PlacementConstraintToProtoConverter(
+ PlacementConstraint placementConstraint) {
+ this.placementConstraint = placementConstraint;
+ }
+
+ public PlacementConstraintProto convert() {
+ return (PlacementConstraintProto) placementConstraint.getConstraintExpr()
+ .accept(this);
+ }
+
+ @Override
+ public GeneratedMessage visit(SingleConstraint constraint) {
+ SimplePlacementConstraintProto.Builder sb =
+ SimplePlacementConstraintProto.newBuilder();
+
+ if (constraint.getScope() != null) {
+ sb.setScope(constraint.getScope());
+ }
+ sb.setMinCardinality(constraint.getMinCardinality());
+ sb.setMaxCardinality(constraint.getMaxCardinality());
+ if (constraint.getTargetExpressions() != null) {
+ for (TargetExpression target : constraint.getTargetExpressions()) {
+ sb.addTargetExpressions(
+ (PlacementConstraintTargetProto) target.accept(this));
+ }
+
+ }
+ SimplePlacementConstraintProto sProto = sb.build();
+
+ // Wrap around PlacementConstraintProto object.
+ PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder();
+ pb.setSimpleConstraint(sProto);
+ return pb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(TargetExpression target) {
+ PlacementConstraintTargetProto.Builder tb =
+ PlacementConstraintTargetProto.newBuilder();
+
+ tb.setTargetType(ProtoUtils.convertToProtoFormat(target.getTargetType()));
+ if (target.getTargetKey() != null) {
+ tb.setTargetKey(target.getTargetKey());
+ }
+ if (target.getTargetValues() != null) {
+ tb.addAllTargetValues(target.getTargetValues());
+ }
+ return tb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(TargetConstraint constraint) {
+ throw new YarnRuntimeException("Unexpected TargetConstraint found.");
+ }
+
+ @Override
+ public GeneratedMessage visit(CardinalityConstraint constraint) {
+ throw new YarnRuntimeException("Unexpected CardinalityConstraint found.");
+ }
+
+ private GeneratedMessage visitAndOr(
+ CompositeConstraint<AbstractConstraint> composite, CompositeType type) {
+ CompositePlacementConstraintProto.Builder cb =
+ CompositePlacementConstraintProto.newBuilder();
+
+ cb.setCompositeType(type);
+
+ for (AbstractConstraint c : composite.getChildren()) {
+ cb.addChildConstraints((PlacementConstraintProto) c.accept(this));
+ }
+ CompositePlacementConstraintProto cProto = cb.build();
+
+ // Wrap around PlacementConstraintProto object.
+ PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder();
+ pb.setCompositeConstraint(cProto);
+ return pb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(And constraint) {
+ return visitAndOr(constraint, CompositeType.AND);
+ }
+
+ @Override
+ public GeneratedMessage visit(Or constraint) {
+ return visitAndOr(constraint, CompositeType.OR);
+ }
+
+ @Override
+ public GeneratedMessage visit(DelayedOr constraint) {
+ CompositePlacementConstraintProto.Builder cb =
+ CompositePlacementConstraintProto.newBuilder();
+
+ cb.setCompositeType(CompositeType.DELAYED_OR);
+
+ for (TimedPlacementConstraint c : constraint.getChildren()) {
+ cb.addTimedChildConstraints(
+ (TimedPlacementConstraintProto) c.accept(this));
+ }
+ CompositePlacementConstraintProto cProto = cb.build();
+
+ // Wrap around PlacementConstraintProto object.
+ PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder();
+ pb.setCompositeConstraint(cProto);
+ return pb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(TimedPlacementConstraint constraint) {
+ TimedPlacementConstraintProto.Builder tb =
+ TimedPlacementConstraintProto.newBuilder();
+
+ tb.setDelayUnit(ProtoUtils.convertToProtoFormat(constraint.getDelayUnit()));
+ tb.setSchedulingDelay(constraint.getSchedulingDelay());
+ tb.setPlacementConstraint(
+ (PlacementConstraintProto) constraint.getConstraint().accept(this));
+
+ return tb.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
new file mode 100644
index 0000000..18da80f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * API related to protobuf objects that are not backed by PBImpl classes.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.api.pb;
+import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 528cf8e..18075f52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -55,6 +55,8 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
@@ -69,10 +71,12 @@ import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestInterpreterProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto;
@@ -495,6 +499,29 @@ public class ProtoUtils {
}
return ret;
}
+
+ public static PlacementConstraintTargetProto.TargetType convertToProtoFormat(
+ TargetExpression.TargetType t) {
+ return PlacementConstraintTargetProto.TargetType.valueOf(t.name());
+ }
+
+ public static TargetExpression.TargetType convertFromProtoFormat(
+ PlacementConstraintTargetProto.TargetType t) {
+ return TargetExpression.TargetType.valueOf(t.name());
+ }
+
+ /*
+ * TimedPlacementConstraint.DelayUnit
+ */
+ public static TimedPlacementConstraintProto.DelayUnit convertToProtoFormat(
+ TimedPlacementConstraint.DelayUnit u) {
+ return TimedPlacementConstraintProto.DelayUnit.valueOf(u.name());
+ }
+
+ public static TimedPlacementConstraint.DelayUnit convertFromProtoFormat(
+ TimedPlacementConstraintProto.DelayUnit u) {
+ return TimedPlacementConstraint.DelayUnit.valueOf(u.name());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java
new file mode 100644
index 0000000..e9eda6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.ListIterator;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint.TargetOperator;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+/**
+ * This class contains inner classes that define transformation on a
+ * {@link PlacementConstraint} expression.
+ */
+@Private
+public class PlacementConstraintTransformations {
+
+ /**
+ * The default implementation of the {@link PlacementConstraint.Visitor} that
+ * does a traversal of the constraint tree, performing no action for the lead
+ * constraints.
+ */
+ public static class AbstractTransformer
+ implements PlacementConstraint.Visitor<AbstractConstraint> {
+
+ private PlacementConstraint placementConstraint;
+
+ public AbstractTransformer(PlacementConstraint placementConstraint) {
+ this.placementConstraint = placementConstraint;
+ }
+
+ /**
+ * This method performs the transformation of the
+ * {@link #placementConstraint}.
+ *
+ * @return the transformed placement constraint.
+ */
+ public PlacementConstraint transform() {
+ AbstractConstraint constraintExpr =
+ placementConstraint.getConstraintExpr();
+
+ // Visit the constraint tree to perform the transformation.
+ constraintExpr = constraintExpr.accept(this);
+
+ return new PlacementConstraint(constraintExpr);
+ }
+
+ @Override
+ public AbstractConstraint visit(SingleConstraint constraint) {
+ // Do nothing.
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(TargetExpression expression) {
+ // Do nothing.
+ return null;
+ }
+
+ @Override
+ public AbstractConstraint visit(TargetConstraint constraint) {
+ // Do nothing.
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(CardinalityConstraint constraint) {
+ // Do nothing.
+ return constraint;
+ }
+
+ private AbstractConstraint visitAndOr(
+ CompositeConstraint<AbstractConstraint> constraint) {
+ for (ListIterator<AbstractConstraint> iter =
+ constraint.getChildren().listIterator(); iter.hasNext();) {
+ AbstractConstraint child = iter.next();
+ child = child.accept(this);
+ iter.set(child);
+ }
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(And constraint) {
+ return visitAndOr(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(Or constraint) {
+ return visitAndOr(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(DelayedOr constraint) {
+ constraint.getChildren().forEach(
+ child -> child.setConstraint(child.getConstraint().accept(this)));
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(TimedPlacementConstraint constraint) {
+ // Do nothing.
+ return null;
+ }
+ }
+
+ /**
+ * Visits a {@link PlacementConstraint} tree and substitutes each
+ * {@link TargetConstraint} and {@link CardinalityConstraint} with an
+ * equivalent {@link SingleConstraint}.
+ */
+ public static class SingleConstraintTransformer extends AbstractTransformer {
+
+ public SingleConstraintTransformer(PlacementConstraint constraint) {
+ super(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(TargetConstraint constraint) {
+ AbstractConstraint newConstraint;
+ if (constraint.getOp() == TargetOperator.IN) {
+ newConstraint = new SingleConstraint(constraint.getScope(), 1,
+ Integer.MAX_VALUE, constraint.getTargetExpressions());
+ } else if (constraint.getOp() == TargetOperator.NOT_IN) {
+ newConstraint = new SingleConstraint(constraint.getScope(), 0, 0,
+ constraint.getTargetExpressions());
+ } else {
+ throw new YarnRuntimeException(
+ "Encountered unexpected type of constraint target operator: "
+ + constraint.getOp());
+ }
+ return newConstraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(CardinalityConstraint constraint) {
+ return new SingleConstraint(constraint.getScope(),
+ constraint.getMinCardinality(), constraint.getMaxCardinality(),
+ new TargetExpression(TargetExpression.TargetType.SELF));
+ }
+ }
+
+ /**
+ * Visits a {@link PlacementConstraint} tree and, whenever possible,
+ * substitutes each {@link SingleConstraint} with a {@link TargetConstraint}
+ * or a {@link CardinalityConstraint}. When such a substitution is not
+ * possible, we keep the original {@link SingleConstraint}.
+ */
+ public static class SpecializedConstraintTransformer
+ extends AbstractTransformer {
+
+ public SpecializedConstraintTransformer(PlacementConstraint constraint) {
+ super(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(SingleConstraint constraint) {
+ AbstractConstraint transformedConstraint = constraint;
+ // Check if it is a cardinality constraint.
+ if (constraint.getTargetExpressions().size() == 1) {
+ TargetExpression targetExpr =
+ constraint.getTargetExpressions().iterator().next();
+ if (targetExpr.getTargetType() == TargetExpression.TargetType.SELF) {
+ transformedConstraint = new CardinalityConstraint(
+ constraint.getScope(), constraint.getMinCardinality(),
+ constraint.getMaxCardinality());
+ }
+ }
+ // Check if it is a target constraint.
+ if (constraint.getMinCardinality() == 1
+ && constraint.getMaxCardinality() == Integer.MAX_VALUE) {
+ transformedConstraint = new TargetConstraint(TargetOperator.IN,
+ constraint.getScope(), constraint.getTargetExpressions());
+ } else if (constraint.getMinCardinality() == 0
+ && constraint.getMaxCardinality() == 0) {
+ transformedConstraint = new TargetConstraint(TargetOperator.NOT_IN,
+ constraint.getScope(), constraint.getTargetExpressions());
+ }
+
+ return transformedConstraint;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
new file mode 100644
index 0000000..660dc02
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * API related to resources.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.api.resource;
+import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
new file mode 100644
index 0000000..bd245e2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+
+import java.util.Iterator;
+
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintFromProtoConverter;
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintToProtoConverter;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test class for {@link PlacementConstraintToProtoConverter} and
+ * {@link PlacementConstraintFromProtoConverter}.
+ */
+public class TestPlacementConstraintPBConversion {
+
+ @Test
+ public void testTargetConstraintProtoConverter() {
+ AbstractConstraint sConstraintExpr =
+ targetIn(NODE, allocationTag("hbase-m"));
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Convert to proto.
+ PlacementConstraintToProtoConverter toProtoConverter =
+ new PlacementConstraintToProtoConverter(sConstraint);
+ PlacementConstraintProto protoConstraint = toProtoConverter.convert();
+
+ Assert.assertTrue(protoConstraint.hasSimpleConstraint());
+ Assert.assertFalse(protoConstraint.hasCompositeConstraint());
+ SimplePlacementConstraintProto sProto =
+ protoConstraint.getSimpleConstraint();
+ Assert.assertEquals(single.getScope(), sProto.getScope());
+ Assert.assertEquals(single.getMinCardinality(), sProto.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(), sProto.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions().size(),
+ sProto.getTargetExpressionsList().size());
+
+ // Convert from proto.
+ PlacementConstraintFromProtoConverter fromProtoConverter =
+ new PlacementConstraintFromProtoConverter(protoConstraint);
+ PlacementConstraint newConstraint = fromProtoConverter.convert();
+
+ AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
+ Assert.assertTrue(newConstraintExpr instanceof SingleConstraint);
+ SingleConstraint newSingle = (SingleConstraint) newConstraintExpr;
+ Assert.assertEquals(single.getScope(), newSingle.getScope());
+ Assert.assertEquals(single.getMinCardinality(),
+ newSingle.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(),
+ newSingle.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions(),
+ newSingle.getTargetExpressions());
+ }
+
+ @Test
+ public void testCardinalityConstraintProtoConverter() {
+ AbstractConstraint sConstraintExpr = cardinality(RACK, 3, 10);
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Convert to proto.
+ PlacementConstraintToProtoConverter toProtoConverter =
+ new PlacementConstraintToProtoConverter(sConstraint);
+ PlacementConstraintProto protoConstraint = toProtoConverter.convert();
+
+ compareSimpleConstraintToProto(single, protoConstraint);
+
+ // Convert from proto.
+ PlacementConstraintFromProtoConverter fromProtoConverter =
+ new PlacementConstraintFromProtoConverter(protoConstraint);
+ PlacementConstraint newConstraint = fromProtoConverter.convert();
+
+ AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
+ Assert.assertTrue(newConstraintExpr instanceof SingleConstraint);
+ SingleConstraint newSingle = (SingleConstraint) newConstraintExpr;
+ compareSimpleConstraints(single, newSingle);
+ }
+
+ @Test
+ public void testCompositeConstraintProtoConverter() {
+ AbstractConstraint constraintExpr =
+ or(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
+ targetCardinality(RACK, 2, 10, allocationTag("zk")));
+ Assert.assertTrue(constraintExpr instanceof Or);
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+ Or orExpr = (Or) constraintExpr;
+
+ // Convert to proto.
+ PlacementConstraintToProtoConverter toProtoConverter =
+ new PlacementConstraintToProtoConverter(constraint);
+ PlacementConstraintProto protoConstraint = toProtoConverter.convert();
+
+ Assert.assertFalse(protoConstraint.hasSimpleConstraint());
+ Assert.assertTrue(protoConstraint.hasCompositeConstraint());
+ CompositePlacementConstraintProto cProto =
+ protoConstraint.getCompositeConstraint();
+
+ Assert.assertEquals(CompositeType.OR, cProto.getCompositeType());
+ Assert.assertEquals(3, cProto.getChildConstraintsCount());
+ Assert.assertEquals(0, cProto.getTimedChildConstraintsCount());
+ Iterator<AbstractConstraint> orChildren = orExpr.getChildren().iterator();
+ Iterator<PlacementConstraintProto> orProtoChildren =
+ cProto.getChildConstraintsList().iterator();
+ while (orChildren.hasNext() && orProtoChildren.hasNext()) {
+ AbstractConstraint orChild = orChildren.next();
+ PlacementConstraintProto orProtoChild = orProtoChildren.next();
+ compareSimpleConstraintToProto((SingleConstraint) orChild, orProtoChild);
+ }
+
+ // Convert from proto.
+ PlacementConstraintFromProtoConverter fromProtoConverter =
+ new PlacementConstraintFromProtoConverter(protoConstraint);
+ PlacementConstraint newConstraint = fromProtoConverter.convert();
+
+ AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
+ Assert.assertTrue(newConstraintExpr instanceof Or);
+ Or newOrExpr = (Or) newConstraintExpr;
+ Assert.assertEquals(3, newOrExpr.getChildren().size());
+ orChildren = orExpr.getChildren().iterator();
+ Iterator<AbstractConstraint> newOrChildren =
+ newOrExpr.getChildren().iterator();
+ while (orChildren.hasNext() && newOrChildren.hasNext()) {
+ AbstractConstraint orChild = orChildren.next();
+ AbstractConstraint newOrChild = newOrChildren.next();
+ compareSimpleConstraints((SingleConstraint) orChild,
+ (SingleConstraint) newOrChild);
+ }
+ }
+
+ private void compareSimpleConstraintToProto(SingleConstraint constraint,
+ PlacementConstraintProto proto) {
+ Assert.assertTrue(proto.hasSimpleConstraint());
+ Assert.assertFalse(proto.hasCompositeConstraint());
+ SimplePlacementConstraintProto sProto = proto.getSimpleConstraint();
+ Assert.assertEquals(constraint.getScope(), sProto.getScope());
+ Assert.assertEquals(constraint.getMinCardinality(),
+ sProto.getMinCardinality());
+ Assert.assertEquals(constraint.getMaxCardinality(),
+ sProto.getMaxCardinality());
+ Assert.assertEquals(constraint.getTargetExpressions().size(),
+ sProto.getTargetExpressionsList().size());
+ }
+
+ private void compareSimpleConstraints(SingleConstraint single,
+ SingleConstraint newSingle) {
+ Assert.assertEquals(single.getScope(), newSingle.getScope());
+ Assert.assertEquals(single.getMinCardinality(),
+ newSingle.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(),
+ newSingle.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions(),
+ newSingle.getTargetExpressions());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/229680a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
new file mode 100644
index 0000000..1763735
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint.TargetOperator;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SpecializedConstraintTransformer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test class for {@link PlacementConstraintTransformations}.
+ */
+public class TestPlacementConstraintTransformations {
+
+ @Test
+ public void testTargetConstraint() {
+ AbstractConstraint sConstraintExpr =
+ targetIn(NODE, allocationTag("hbase-m"));
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Transform from SimpleConstraint to specialized TargetConstraint
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(sConstraint);
+ PlacementConstraint tConstraint = specTransformer.transform();
+
+ AbstractConstraint tConstraintExpr = tConstraint.getConstraintExpr();
+ Assert.assertTrue(tConstraintExpr instanceof TargetConstraint);
+
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ TargetConstraint target = (TargetConstraint) tConstraintExpr;
+ Assert.assertEquals(single.getScope(), target.getScope());
+ Assert.assertEquals(TargetOperator.IN, target.getOp());
+ Assert.assertEquals(single.getTargetExpressions(),
+ target.getTargetExpressions());
+
+ // Transform from specialized TargetConstraint to SimpleConstraint
+ SingleConstraintTransformer singleTransformer =
+ new SingleConstraintTransformer(tConstraint);
+ sConstraint = singleTransformer.transform();
+
+ sConstraintExpr = sConstraint.getConstraintExpr();
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+
+ single = (SingleConstraint) sConstraintExpr;
+ Assert.assertEquals(target.getScope(), single.getScope());
+ Assert.assertEquals(1, single.getMinCardinality());
+ Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions(),
+ target.getTargetExpressions());
+ }
+
+ @Test
+ public void testCardinalityConstraint() {
+ AbstractConstraint sConstraintExpr = cardinality(RACK, 3, 10);
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Transform from SimpleConstraint to specialized CardinalityConstraint
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(sConstraint);
+ PlacementConstraint cConstraint = specTransformer.transform();
+
+ AbstractConstraint cConstraintExpr = cConstraint.getConstraintExpr();
+ Assert.assertTrue(cConstraintExpr instanceof CardinalityConstraint);
+
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ CardinalityConstraint cardinality = (CardinalityConstraint) cConstraintExpr;
+ Assert.assertEquals(single.getScope(), cardinality.getScope());
+ Assert.assertEquals(single.getMinCardinality(),
+ cardinality.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(),
+ cardinality.getMaxCardinality());
+
+ // Transform from specialized CardinalityConstraint to SimpleConstraint
+ SingleConstraintTransformer singleTransformer =
+ new SingleConstraintTransformer(cConstraint);
+ sConstraint = singleTransformer.transform();
+
+ sConstraintExpr = sConstraint.getConstraintExpr();
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+
+ single = (SingleConstraint) sConstraintExpr;
+ Assert.assertEquals(cardinality.getScope(), single.getScope());
+ Assert.assertEquals(cardinality.getMinCardinality(),
+ single.getMinCardinality());
+ Assert.assertEquals(cardinality.getMaxCardinality(),
+ single.getMaxCardinality());
+ Assert.assertEquals(new HashSet<>(Arrays.asList(PlacementTargets.self())),
+ single.getTargetExpressions());
+ }
+
+ @Test
+ public void testTargetCardinalityConstraint() {
+ AbstractConstraint constraintExpr =
+ targetCardinality(RACK, 3, 10, allocationTag("zk"));
+ Assert.assertTrue(constraintExpr instanceof SingleConstraint);
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+
+ // Apply transformation. Should be a no-op.
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(constraint);
+ PlacementConstraint newConstraint = specTransformer.transform();
+
+ // The constraint expression should be the same.
+ Assert.assertEquals(constraintExpr, newConstraint.getConstraintExpr());
+ }
+
+ @Test
+ public void testCompositeConstraint() {
+ AbstractConstraint constraintExpr =
+ or(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
+ targetCardinality(RACK, 2, 10, allocationTag("zk")));
+ Assert.assertTrue(constraintExpr instanceof Or);
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+ Or orExpr = (Or) constraintExpr;
+ for (AbstractConstraint child : orExpr.getChildren()) {
+ Assert.assertTrue(child instanceof SingleConstraint);
+ }
+
+ // Apply transformation. Should transform target and cardinality constraints
+ // included in the composite constraint to specialized ones.
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(constraint);
+ PlacementConstraint specConstraint = specTransformer.transform();
+
+ Or specOrExpr = (Or) specConstraint.getConstraintExpr();
+ List<AbstractConstraint> specChildren = specOrExpr.getChildren();
+ Assert.assertEquals(3, specChildren.size());
+ Assert.assertTrue(specChildren.get(0) instanceof TargetConstraint);
+ Assert.assertTrue(specChildren.get(1) instanceof CardinalityConstraint);
+ Assert.assertTrue(specChildren.get(2) instanceof SingleConstraint);
+
+ // Transform from specialized TargetConstraint to SimpleConstraint
+ SingleConstraintTransformer singleTransformer =
+ new SingleConstraintTransformer(specConstraint);
+ PlacementConstraint simConstraint = singleTransformer.transform();
+ Assert.assertTrue(constraintExpr instanceof Or);
+ Or simOrExpr = (Or) specConstraint.getConstraintExpr();
+ for (AbstractConstraint child : simOrExpr.getChildren()) {
+ Assert.assertTrue(child instanceof SingleConstraint);
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: [HDFS-12386] Add fsserver defaults
call to WebhdfsFileSystem. (Rushabh Shah via daryn)
Posted by kk...@apache.org.
[HDFS-12386] Add fsserver defaults call to WebhdfsFileSystem. (Rushabh Shah via daryn)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0da29cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0da29cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0da29cbe
Branch: refs/heads/YARN-6592
Commit: 0da29cbeea40cb7839abcd72566b997962829329
Parents: 9df0500
Author: Daryn Sharp <da...@yahoo-inc.com>
Authored: Tue Sep 26 15:15:24 2017 -0500
Committer: Daryn Sharp <da...@yahoo-inc.com>
Committed: Tue Sep 26 15:15:24 2017 -0500
----------------------------------------------------------------------
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 38 ++++++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 17 +++
.../hadoop/hdfs/web/resources/GetOpParam.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 21 +++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 20 +++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 134 +++++++++++++++++++
7 files changed, 234 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da29cbe/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index dcd73bf..53d886d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.ContentSummary.Builder;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
@@ -65,6 +66,8 @@ import java.util.Map;
class JsonUtilClient {
static final DatanodeInfo[] EMPTY_DATANODE_INFO_ARRAY = {};
+ static final String UNSUPPPORTED_EXCEPTION_STR =
+ UnsupportedOperationException.class.getName();
/** Convert a Json map to a RemoteException. */
static RemoteException toRemoteException(final Map<?, ?> json) {
@@ -72,6 +75,9 @@ class JsonUtilClient {
RemoteException.class.getSimpleName());
final String message = (String)m.get("message");
final String javaClassName = (String)m.get("javaClassName");
+ if (UNSUPPPORTED_EXCEPTION_STR.equals(javaClassName)) {
+ throw new UnsupportedOperationException(message);
+ }
return new RemoteException(javaClassName, message);
}
@@ -644,4 +650,36 @@ class JsonUtilClient {
}
}
+ /*
+ * The parameters which have default value -1 are required fields according
+ * to hdfs.proto.
+ * The default values for optional fields are taken from
+ * hdfs.proto#FsServerDefaultsProto.
+ */
+ public static FsServerDefaults toFsServerDefaults(final Map<?, ?> json) {
+ if (json == null) {
+ return null;
+ }
+ Map<?, ?> m =
+ (Map<?, ?>) json.get(FsServerDefaults.class.getSimpleName());
+ long blockSize = getLong(m, "blockSize", -1);
+ int bytesPerChecksum = getInt(m, "bytesPerChecksum", -1);
+ int writePacketSize = getInt(m, "writePacketSize", -1);
+ short replication = (short) getInt(m, "replication", -1);
+ int fileBufferSize = getInt(m, "fileBufferSize", -1);
+ boolean encryptDataTransfer = m.containsKey("encryptDataTransfer")
+ ? (Boolean) m.get("encryptDataTransfer")
+ : false;
+ long trashInterval = getLong(m, "trashInterval", 0);
+ DataChecksum.Type type =
+ DataChecksum.Type.valueOf(getInt(m, "checksumType", 1));
+ String keyProviderUri = (String) m.get("keyProviderUri");
+ byte storagepolicyId = m.containsKey("defaultStoragePolicyId")
+ ? ((Number) m.get("defaultStoragePolicyId")).byteValue()
+ : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
+ return new FsServerDefaults(blockSize, bytesPerChecksum,
+ writePacketSize, replication, fileBufferSize,
+ encryptDataTransfer, trashInterval, type, keyProviderUri,
+ storagepolicyId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da29cbe/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index ee8d5c1..d3a8b23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
import org.apache.hadoop.fs.StorageStatistics;
@@ -1766,6 +1767,22 @@ public class WebHdfsFileSystem extends FileSystem
new FsPathRunner(op, src).run();
}
+ /*
+ * Caller of this method should handle UnsupportedOperationException in case
+ * when new client is talking to old namenode that don't support
+ * FsServerDefaults call.
+ */
+ @Override
+ public FsServerDefaults getServerDefaults() throws IOException {
+ final HttpOpParam.Op op = GetOpParam.Op.GETSERVERDEFAULTS;
+ return new FsPathResponseRunner<FsServerDefaults>(op, null) {
+ @Override
+ FsServerDefaults decodeResponse(Map<?, ?> json) throws IOException {
+ return JsonUtilClient.toFsServerDefaults(json);
+ }
+ }.run();
+ }
+
@VisibleForTesting
InetSocketAddress[] getResolvedNNAddr() {
return nnAddrs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da29cbe/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index 6c2c674..4d4d9be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -46,7 +46,8 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED),
CHECKACCESS(false, HttpURLConnection.HTTP_OK),
- LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK);
+ LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK),
+ GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK);
final boolean redirect;
final int expectedHttpResponseCode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da29cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1534850..4094694 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1782,7 +1782,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return sw.toString();
}
- FsServerDefaults getServerDefaults() throws StandbyException {
+ @VisibleForTesting
+ public FsServerDefaults getServerDefaults() throws StandbyException {
checkOperation(OperationCategory.READ);
return serverDefaults;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da29cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index e400847..9e0a1ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclStatus;
@@ -115,6 +116,7 @@ public class NamenodeWebHdfsMethods {
private Principal userPrincipal;
private String remoteAddr;
+ private static volatile String serverDefaultsResponse = null;
private @Context ServletContext context;
private @Context HttpServletResponse response;
@@ -1121,11 +1123,30 @@ public class NamenodeWebHdfsMethods {
final String js = JsonUtil.toJsonString(storagePolicy);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
+ case GETSERVERDEFAULTS: {
+ // Since none of the server defaults values are hot reloaded, we can
+ // cache the output of serverDefaults.
+ if (serverDefaultsResponse == null) {
+ FsServerDefaults serverDefaults = np.getServerDefaults();
+ serverDefaultsResponse = JsonUtil.toJsonString(serverDefaults);
+ }
+ return Response.ok(serverDefaultsResponse)
+ .type(MediaType.APPLICATION_JSON).build();
+ }
default:
throw new UnsupportedOperationException(op + " is not supported");
}
}
+ /*
+ * This is used only and only for testing.
+ * Please don't use it otherwise.
+ */
+ @VisibleForTesting
+ public static void resetServerDefaultsResponse() {
+ serverDefaultsResponse = null;
+ }
+
private static String getTrashRoot(String fullPath,
Configuration conf) throws IOException {
FileSystem fs = FileSystem.get(conf != null ? conf : new Configuration());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da29cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 5752948..7fa818a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.XAttr;
@@ -470,4 +471,23 @@ public class JsonUtil {
public static String toJsonString(BlockStoragePolicy storagePolicy) {
return toJsonString(BlockStoragePolicy.class, toJsonMap(storagePolicy));
}
+
+ public static String toJsonString(FsServerDefaults serverDefaults) {
+ return toJsonString(FsServerDefaults.class, toJsonMap(serverDefaults));
+ }
+
+ private static Object toJsonMap(FsServerDefaults serverDefaults) {
+ final Map<String, Object> m = new HashMap<String, Object>();
+ m.put("blockSize", serverDefaults.getBlockSize());
+ m.put("bytesPerChecksum", serverDefaults.getBytesPerChecksum());
+ m.put("writePacketSize", serverDefaults.getWritePacketSize());
+ m.put("replication", serverDefaults.getReplication());
+ m.put("fileBufferSize", serverDefaults.getFileBufferSize());
+ m.put("encryptDataTransfer", serverDefaults.getEncryptDataTransfer());
+ m.put("trashInterval", serverDefaults.getTrashInterval());
+ m.put("checksumType", serverDefaults.getChecksumType().id);
+ m.put("keyProviderUri", serverDefaults.getKeyProviderUri());
+ m.put("defaultStoragePolicyId", serverDefaults.getDefaultStoragePolicyId());
+ return m;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da29cbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index de051b3..3ee8ad0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -18,6 +18,14 @@
package org.apache.hadoop.hdfs.web;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -53,6 +61,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
@@ -74,7 +83,9 @@ import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
@@ -92,11 +103,13 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.log4j.Level;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
import static org.mockito.Matchers.any;
@@ -1318,4 +1331,125 @@ public class TestWebHDFS {
}
}
}
+
+ /**
+ * Test fsserver defaults response from {@link DistributedFileSystem} and
+ * {@link WebHdfsFileSystem} are the same.
+ * @throws Exception
+ */
+ @Test
+ public void testFsserverDefaults() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ // Here we override all the default values so that we can verify that it
+ // doesn't pick up the default value.
+ long blockSize = 256*1024*1024;
+ int bytesPerChecksum = 256;
+ int writePacketSize = 128*1024;
+ int replicationFactor = 0;
+ int bufferSize = 1024;
+ boolean encryptDataTransfer = true;
+ long trashInterval = 1;
+ String checksumType = "CRC32";
+ // Setting policy to a special value 7 because BlockManager will
+ // create defaultSuite with policy id 7.
+ byte policyId = (byte) 7;
+
+ conf.setLong(DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setInt(DFS_BYTES_PER_CHECKSUM_KEY, bytesPerChecksum);
+ conf.setInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, writePacketSize);
+ conf.setInt(DFS_REPLICATION_KEY, replicationFactor);
+ conf.setInt(IO_FILE_BUFFER_SIZE_KEY, bufferSize);
+ conf.setBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, encryptDataTransfer);
+ conf.setLong(FS_TRASH_INTERVAL_KEY, trashInterval);
+ conf.set(DFS_CHECKSUM_TYPE_KEY, checksumType);
+ FsServerDefaults originalServerDefaults = new FsServerDefaults(blockSize,
+ bytesPerChecksum, writePacketSize, (short)replicationFactor,
+ bufferSize, encryptDataTransfer, trashInterval,
+ DataChecksum.Type.valueOf(checksumType), "", policyId);
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ FsServerDefaults dfsServerDefaults = dfs.getServerDefaults();
+ FsServerDefaults webfsServerDefaults = webfs.getServerDefaults();
+ // Verify whether server defaults value that we override is equal to
+ // dfsServerDefaults.
+ compareFsServerDefaults(originalServerDefaults, dfsServerDefaults);
+ // Verify whether dfs serverdefaults is equal to
+ // webhdfsServerDefaults.
+ compareFsServerDefaults(dfsServerDefaults, webfsServerDefaults);
+ webfs.getServerDefaults();
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private void compareFsServerDefaults(FsServerDefaults serverDefaults1,
+ FsServerDefaults serverDefaults2) throws Exception {
+ Assert.assertEquals("Block size is different",
+ serverDefaults1.getBlockSize(),
+ serverDefaults2.getBlockSize());
+ Assert.assertEquals("Bytes per checksum are different",
+ serverDefaults1.getBytesPerChecksum(),
+ serverDefaults2.getBytesPerChecksum());
+ Assert.assertEquals("Write packet size is different",
+ serverDefaults1.getWritePacketSize(),
+ serverDefaults2.getWritePacketSize());
+ Assert.assertEquals("Default replication is different",
+ serverDefaults1.getReplication(),
+ serverDefaults2.getReplication());
+ Assert.assertEquals("File buffer size are different",
+ serverDefaults1.getFileBufferSize(),
+ serverDefaults2.getFileBufferSize());
+ Assert.assertEquals("Encrypt data transfer key is different",
+ serverDefaults1.getEncryptDataTransfer(),
+ serverDefaults2.getEncryptDataTransfer());
+ Assert.assertEquals("Trash interval is different",
+ serverDefaults1.getTrashInterval(),
+ serverDefaults2.getTrashInterval());
+ Assert.assertEquals("Checksum type is different",
+ serverDefaults1.getChecksumType(),
+ serverDefaults2.getChecksumType());
+ Assert.assertEquals("Key provider uri is different",
+ serverDefaults1.getKeyProviderUri(),
+ serverDefaults2.getKeyProviderUri());
+ Assert.assertEquals("Default storage policy is different",
+ serverDefaults1.getDefaultStoragePolicyId(),
+ serverDefaults2.getDefaultStoragePolicyId());
+ }
+
+ /**
+ * Tests the case when client is upgraded to return {@link FsServerDefaults}
+ * but then namenode is not upgraded.
+ * @throws Exception
+ */
+ @Test
+ public void testFsserverDefaultsBackwardsCompatible() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ final WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystem(
+ conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ NamenodeWebHdfsMethods.resetServerDefaultsResponse();
+ FSNamesystem fsnSpy =
+ NameNodeAdapter.spyOnNamesystem(cluster.getNameNode());
+ Mockito.when(fsnSpy.getServerDefaults()).
+ thenThrow(new UnsupportedOperationException());
+ try {
+ webfs.getServerDefaults();
+ Assert.fail("should have thrown UnSupportedOperationException.");
+ } catch (UnsupportedOperationException uoe) {
+ //Expected exception.
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-12576. JournalNodes are
getting started,
even though dfs.namenode.shared.edits.dir is not configured. Contributed by
Bharat Viswanadham.
Posted by kk...@apache.org.
HDFS-12576. JournalNodes are getting started, even though dfs.namenode.shared.edits.dir is not configured. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/015abcd8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/015abcd8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/015abcd8
Branch: refs/heads/YARN-6592
Commit: 015abcd8ce1d360830ee2960f1cdc743e09f1629
Parents: 27ffd43
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Oct 2 15:40:00 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Oct 2 15:40:00 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh | 2 +-
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/015abcd8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
index 12d5209..dbdf41e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
@@ -146,7 +146,7 @@ fi
JOURNAL_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -journalNodes 2>&-)
-if [[ "{$JOURNAL_NODES-}" != $'\n' ]]; then
+if [[ "${#JOURNAL_NODES}" != 0 ]]; then
echo "Starting journal nodes [${JOURNAL_NODES}]"
hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/015abcd8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
index 92974de..3da146e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
@@ -102,7 +102,7 @@ fi
JOURNAL_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -journalNodes 2>&-)
-if [[ "{$JOURNAL_NODES-}" != $'\n' ]]; then
+if [[ "${#JOURNAL_NODES}" != 0 ]]; then
echo "Stopping journal nodes [${JOURNAL_NODES}]"
hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: MAPREDUCE-6971. Moving logging
APIs over to slf4j in hadoop-mapreduce-client-app. Contributed by Jinjiang
Ling.
Posted by kk...@apache.org.
MAPREDUCE-6971. Moving logging APIs over to slf4j in hadoop-mapreduce-client-app. Contributed by Jinjiang Ling.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/453d48bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/453d48bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/453d48bd
Branch: refs/heads/YARN-6592
Commit: 453d48bdfbb67ed3e66c33c4aef239c3d7bdd3bc
Parents: 7eb8499
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Oct 3 12:14:54 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Oct 3 12:14:54 2017 +0900
----------------------------------------------------------------------
.../hadoop/mapred/LocalContainerLauncher.java | 13 +++++++------
.../hadoop/mapred/TaskAttemptListenerImpl.java | 11 ++++++-----
.../java/org/apache/hadoop/mapred/YarnChild.java | 19 ++++++++-----------
.../apache/hadoop/mapred/YarnOutputFiles.java | 2 --
.../jobhistory/JobHistoryCopyService.java | 7 ++++---
.../jobhistory/JobHistoryEventHandler.java | 6 +++---
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 15 ++++++---------
.../mapreduce/v2/app/TaskHeartbeatHandler.java | 7 ++++---
.../mapreduce/v2/app/client/MRClientService.java | 6 +++---
.../v2/app/commit/CommitterEventHandler.java | 8 ++++----
.../mapreduce/v2/app/job/impl/JobImpl.java | 6 +++---
.../v2/app/job/impl/TaskAttemptImpl.java | 7 ++++---
.../mapreduce/v2/app/job/impl/TaskImpl.java | 6 +++---
.../v2/app/launcher/ContainerLauncherImpl.java | 7 ++++---
.../v2/app/local/LocalContainerAllocator.java | 8 ++++----
.../mapreduce/v2/app/rm/RMCommunicator.java | 7 ++++---
.../v2/app/rm/RMContainerAllocator.java | 6 +++---
.../v2/app/rm/RMContainerRequestor.java | 7 ++++---
.../preemption/CheckpointAMPreemptionPolicy.java | 8 ++++----
.../rm/preemption/KillAMPreemptionPolicy.java | 8 ++++----
.../v2/app/speculate/DefaultSpeculator.java | 7 ++++---
.../mapreduce/v2/app/webapp/AppController.java | 7 ++++---
.../mapred/TestLocalContainerLauncher.java | 8 ++++----
.../jobhistory/TestJobHistoryEventHandler.java | 8 ++++----
.../mapreduce/jobhistory/TestJobSummary.java | 7 ++++---
.../apache/hadoop/mapreduce/v2/app/MRApp.java | 6 +++---
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 8 +++-----
.../hadoop/mapreduce/v2/app/TestMRAppMaster.java | 15 +++++++--------
.../hadoop/mapreduce/v2/app/TestRecovery.java | 6 +++---
.../mapreduce/v2/app/TestRuntimeEstimators.java | 7 ++++---
.../v2/app/job/impl/TestMapReduceChildJVM.java | 7 ++++---
.../mapreduce/v2/app/job/impl/TestTaskImpl.java | 6 +++---
.../v2/app/launcher/TestContainerLauncher.java | 9 +++++----
.../app/launcher/TestContainerLauncherImpl.java | 7 ++++---
.../v2/app/rm/TestRMContainerAllocator.java | 8 ++++----
35 files changed, 142 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 0b942b0..6f9cc34 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -37,8 +37,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -70,6 +68,8 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Runs the container task locally in a thread.
@@ -80,7 +80,8 @@ public class LocalContainerLauncher extends AbstractService implements
ContainerLauncher {
private static final File curDir = new File(".");
- private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalContainerLauncher.class);
private FileContext curFC = null;
private Set<File> localizedFiles = new HashSet<File>();
@@ -376,7 +377,7 @@ public class LocalContainerLauncher extends AbstractService implements
// if umbilical itself barfs (in error-handler of runSubMap()),
// we're pretty much hosed, so do what YarnChild main() does
// (i.e., exit clumsily--but can never happen, so no worries!)
- LOG.fatal("oopsie... this can never happen: "
+ LOG.error("oopsie... this can never happen: "
+ StringUtils.stringifyException(ioe));
ExitUtil.terminate(-1);
} finally {
@@ -477,7 +478,7 @@ public class LocalContainerLauncher extends AbstractService implements
}
} catch (FSError e) {
- LOG.fatal("FSError from child", e);
+ LOG.error("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
if (!ShutdownHookManager.get().isShutdownInProgress()) {
umbilical.fsError(classicAttemptID, e.getMessage());
@@ -502,7 +503,7 @@ public class LocalContainerLauncher extends AbstractService implements
throw new RuntimeException();
} catch (Throwable throwable) {
- LOG.fatal("Error running local (uberized) 'child' : "
+ LOG.error("Error running local (uberized) 'child' : "
+ StringUtils.stringifyException(throwable));
if (!ShutdownHookManager.get().isShutdownInProgress()) {
Throwable tCause = throwable.getCause();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index 5669f3e..9b6148c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -26,8 +26,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.ProtocolSignature;
@@ -57,6 +55,8 @@ import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class is responsible for talking to the task umblical.
@@ -72,7 +72,8 @@ public class TaskAttemptListenerImpl extends CompositeService
private static final JvmTask TASK_FOR_INVALID_JVM = new JvmTask(null, true);
- private static final Log LOG = LogFactory.getLog(TaskAttemptListenerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskAttemptListenerImpl.class);
private AppContext context;
private Server server;
@@ -274,7 +275,7 @@ public class TaskAttemptListenerImpl extends CompositeService
public void fatalError(TaskAttemptID taskAttemptID, String msg)
throws IOException {
// This happens only in Child and in the Task.
- LOG.fatal("Task: " + taskAttemptID + " - exited : " + msg);
+ LOG.error("Task: " + taskAttemptID + " - exited : " + msg);
reportDiagnosticInfo(taskAttemptID, "Error: " + msg);
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
@@ -291,7 +292,7 @@ public class TaskAttemptListenerImpl extends CompositeService
public void fsError(TaskAttemptID taskAttemptID, String message)
throws IOException {
// This happens only in Child.
- LOG.fatal("Task: " + taskAttemptID + " - failed due to FSError: "
+ LOG.error("Task: " + taskAttemptID + " - failed due to FSError: "
+ message);
reportDiagnosticInfo(taskAttemptID, "FSError: " + message);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 97642a5..7ae7a1e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -26,8 +26,6 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.ScheduledExecutorService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
@@ -60,13 +58,15 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The main() for MapReduce task processes.
*/
class YarnChild {
- private static final Log LOG = LogFactory.getLog(YarnChild.class);
+ private static final Logger LOG = LoggerFactory.getLogger(YarnChild.class);
static volatile TaskAttemptID taskid = null;
@@ -100,10 +100,7 @@ class YarnChild {
// Security framework already loaded the tokens into current ugi
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
- LOG.info("Executing with tokens:");
- for (Token<?> token: credentials.getAllTokens()) {
- LOG.info(token);
- }
+ LOG.info("Executing with tokens: {}", credentials.getAllTokens());
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
@@ -129,7 +126,7 @@ class YarnChild {
try {
int idleLoopCount = 0;
- JvmTask myTask = null;;
+ JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle) {
long sleepTimeMilliSecs = Math.min(idle * 500, 1500);
@@ -179,7 +176,7 @@ class YarnChild {
}
});
} catch (FSError e) {
- LOG.fatal("FSError from child", e);
+ LOG.error("FSError from child", e);
if (!ShutdownHookManager.get().isShutdownInProgress()) {
umbilical.fsError(taskid, e.getMessage());
}
@@ -213,7 +210,7 @@ class YarnChild {
}
}
} catch (Throwable throwable) {
- LOG.fatal("Error running child : "
+ LOG.error("Error running child : "
+ StringUtils.stringifyException(throwable));
if (taskid != null) {
if (!ShutdownHookManager.get().isShutdownInProgress()) {
@@ -352,7 +349,7 @@ class YarnChild {
out = FileSystem.create(localFs, jobFile, urw_gr);
conf.writeXml(out);
} finally {
- IOUtils.cleanup(LOG, out);
+ IOUtils.cleanupWithLogger(LOG, out);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
index e099b8f..ebcb3e7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
index 4bb1e38..ee4ec2c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
@@ -34,6 +32,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Reads in history events from the JobHistoryFile and sends them out again
@@ -41,7 +41,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
*/
public class JobHistoryCopyService extends CompositeService implements HistoryEventHandler {
- private static final Log LOG = LogFactory.getLog(JobHistoryCopyService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(JobHistoryCopyService.class);
private final ApplicationAttemptId applicationAttemptId;
private final EventHandler<Event> handler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 8fa417b..ae46129 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -31,8 +31,6 @@ import java.util.TimerTask;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -82,6 +80,8 @@ import org.apache.hadoop.yarn.util.TimelineServiceHelper;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientHandlerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The job history events get routed to this class. This class writes the Job
@@ -123,7 +123,7 @@ public class JobHistoryEventHandler extends AbstractService
private volatile boolean stopped;
private final Object lock = new Object();
- private static final Log LOG = LogFactory.getLog(
+ private static final Logger LOG = LoggerFactory.getLogger(
JobHistoryEventHandler.class);
protected static final Map<JobId, MetaInfo> fileMap =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index f511f19..df61928 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -41,8 +41,6 @@ import java.util.regex.Pattern;
import javax.crypto.KeyGenerator;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -155,6 +153,8 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The Map-Reduce Application Master.
@@ -177,7 +177,7 @@ import com.google.common.annotations.VisibleForTesting;
@SuppressWarnings("rawtypes")
public class MRAppMaster extends CompositeService {
- private static final Log LOG = LogFactory.getLog(MRAppMaster.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MRAppMaster.class);
/**
* Priority of the MRAppMaster shutdown hook.
@@ -327,7 +327,7 @@ public class MRAppMaster extends CompositeService {
errorHappenedShutDown = true;
forcedState = JobStateInternal.ERROR;
shutDownMessage = "Staging dir does not exist " + stagingDir;
- LOG.fatal(shutDownMessage);
+ LOG.error(shutDownMessage);
} else if (commitStarted) {
//A commit was started so this is the last time, we just need to know
// what result we will use to notify, and how we will unregister
@@ -1667,7 +1667,7 @@ public class MRAppMaster extends CompositeService {
conf.set(MRJobConfig.USER_NAME, jobUserName);
initAndStartAppMaster(appMaster, conf, jobUserName);
} catch (Throwable t) {
- LOG.fatal("Error starting MRAppMaster", t);
+ LOG.error("Error starting MRAppMaster", t);
ExitUtil.terminate(1, t);
}
}
@@ -1716,10 +1716,7 @@ public class MRAppMaster extends CompositeService {
// them
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
- LOG.info("Executing with tokens:");
- for (Token<?> token : credentials.getAllTokens()) {
- LOG.info(token);
- }
+ LOG.info("Executing with tokens: {}", credentials.getAllTokens());
UserGroupInformation appMasterUgi = UserGroupInformation
.createRemoteUser(jobUserName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
index 6a716c7..c438b35 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
@@ -24,8 +24,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.util.MRJobConfUtil;
@@ -36,6 +34,8 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -63,7 +63,8 @@ public class TaskHeartbeatHandler extends AbstractService {
}
}
- private static final Log LOG = LogFactory.getLog(TaskHeartbeatHandler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskHeartbeatHandler.class);
//thread which runs periodically to see the last time since a heartbeat is
//received from a task.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
index 6eb42be..b2dc8ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
@@ -23,8 +23,6 @@ import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.http.HttpConfig.Policy;
@@ -90,6 +88,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This module is responsible for talking to the
@@ -98,7 +98,7 @@ import org.apache.hadoop.yarn.webapp.WebApps;
*/
public class MRClientService extends AbstractService implements ClientService {
- static final Log LOG = LogFactory.getLog(MRClientService.class);
+ static final Logger LOG = LoggerFactory.getLogger(MRClientService.class);
private MRClientProtocol protocolHandler;
private Server server;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
index 0b1be70..849c914 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/commit/CommitterEventHandler.java
@@ -26,8 +26,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -54,12 +52,14 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CommitterEventHandler extends AbstractService
implements EventHandler<CommitterEvent> {
- private static final Log LOG =
- LogFactory.getLog(CommitterEventHandler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CommitterEventHandler.class);
private final AppContext context;
private final OutputCommitter committer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index abc3e61..757c545 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -38,8 +38,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -133,6 +131,8 @@ import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** Implementation of Job interface. Maintains the state machines of Job.
* The read and write calls use ReadWriteLock for concurrency.
@@ -147,7 +147,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
private static final TaskCompletionEvent[]
EMPTY_TASK_COMPLETION_EVENTS = new TaskCompletionEvent[0];
- private static final Log LOG = LogFactory.getLog(JobImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(JobImpl.class);
//The maximum fraction of fetch failures allowed for a map
private float maxAllowedFetchFailuresFraction;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 3faad48..650f387 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -38,8 +38,6 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -139,6 +137,8 @@ import org.apache.hadoop.yarn.util.RackResolver;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Implementation of TaskAttempt interface.
@@ -149,7 +149,8 @@ public abstract class TaskAttemptImpl implements
EventHandler<TaskAttemptEvent> {
static final Counters EMPTY_COUNTERS = new Counters();
- private static final Log LOG = LogFactory.getLog(TaskAttemptImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskAttemptImpl.class);
private static final long MEMORY_SPLITS_RESOLUTION = 1024; //TODO Make configurable?
private final static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
index 228ae24..086d4d5 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
@@ -32,8 +32,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Counters;
@@ -92,6 +90,8 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
@@ -101,7 +101,7 @@ import com.google.common.annotations.VisibleForTesting;
@SuppressWarnings({ "rawtypes", "unchecked" })
public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
- private static final Log LOG = LogFactory.getLog(TaskImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TaskImpl.class);
private static final String SPECULATION = "Speculation: ";
protected final JobConf conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
index cb4f0c9..288a743 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
@@ -32,8 +32,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -60,6 +58,8 @@ import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.C
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class is responsible for launching of containers.
@@ -67,7 +67,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ContainerLauncherImpl extends AbstractService implements
ContainerLauncher {
- static final Log LOG = LogFactory.getLog(ContainerLauncherImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLauncherImpl.class);
private ConcurrentHashMap<ContainerId, Container> containers =
new ConcurrentHashMap<ContainerId, Container>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
index 269e5f4..01338b4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2.app.local;
import java.io.IOException;
import java.util.ArrayList;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobCounter;
@@ -55,6 +53,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Allocates containers locally. Doesn't allocate a real container;
@@ -63,8 +63,8 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
public class LocalContainerAllocator extends RMCommunicator
implements ContainerAllocator {
- private static final Log LOG =
- LogFactory.getLog(LocalContainerAllocator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalContainerAllocator.class);
@SuppressWarnings("rawtypes")
private final EventHandler eventHandler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
index a7058e0..7bcb2ce 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
@@ -26,8 +26,6 @@ import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -61,13 +59,16 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Registers/unregisters to RM and sends heartbeats to RM.
*/
public abstract class RMCommunicator extends AbstractService
implements RMHeartbeatHandler {
- private static final Log LOG = LogFactory.getLog(RMCommunicator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RMCommunicator.class);
private int rmPollInterval;//millis
protected ApplicationId applicationId;
private final AtomicBoolean stopped;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index 0dc7642..9d030b8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -35,8 +35,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -90,6 +88,8 @@ import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Allocates the container from the ResourceManager scheduler.
@@ -97,7 +97,7 @@ import com.google.common.annotations.VisibleForTesting;
public class RMContainerAllocator extends RMContainerRequestor
implements ContainerAllocator {
- static final Log LOG = LogFactory.getLog(RMContainerAllocator.class);
+ static final Logger LOG = LoggerFactory.getLogger(RMContainerAllocator.class);
public static final
float DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART = 0.05f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index f4579ab..bb3e1fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -30,8 +30,6 @@ import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -55,6 +53,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -62,7 +62,8 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*/
public abstract class RMContainerRequestor extends RMCommunicator {
- private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RMContainerRequestor.class);
private static final ResourceRequestComparator RESOURCE_REQUEST_COMPARATOR =
new ResourceRequestComparator();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
index b0e399f..6192c4d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/CheckpointAMPreemptionPolicy.java
@@ -25,8 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -44,6 +42,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This policy works in combination with an implementation of task
@@ -75,8 +75,8 @@ public class CheckpointAMPreemptionPolicy implements AMPreemptionPolicy {
@SuppressWarnings("rawtypes")
private EventHandler eventHandler;
- static final Log LOG = LogFactory
- .getLog(CheckpointAMPreemptionPolicy.class);
+ static final Logger LOG = LoggerFactory
+ .getLogger(CheckpointAMPreemptionPolicy.class);
public CheckpointAMPreemptionPolicy() {
this(Collections.synchronizedSet(new HashSet<TaskAttemptId>()),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
index 09237aa..eec6854 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/preemption/KillAMPreemptionPolicy.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.mapreduce.v2.app.rm.preemption;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -33,14 +31,16 @@ import org.apache.hadoop.yarn.api.records.PreemptionContract;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Sample policy that aggressively kills tasks when requested.
*/
public class KillAMPreemptionPolicy implements AMPreemptionPolicy {
- private static final Log LOG =
- LogFactory.getLog(KillAMPreemptionPolicy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(KillAMPreemptionPolicy.class);
@SuppressWarnings("rawtypes")
private EventHandler dispatcher = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
index ed06493..fa65383 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
@@ -30,8 +30,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -53,6 +51,8 @@ import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.event.Event;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DefaultSpeculator extends AbstractService implements
Speculator {
@@ -70,7 +70,8 @@ public class DefaultSpeculator extends AbstractService implements
private double proportionTotalTasksSpeculatable;
private int minimumAllowedSpeculativeTasks;
- private static final Log LOG = LogFactory.getLog(DefaultSpeculator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultSpeculator.class);
private final ConcurrentMap<TaskId, Boolean> runningTasks
= new ConcurrentHashMap<TaskId, Boolean>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
index 6db1274..1ba37c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
@@ -26,8 +26,6 @@ import java.net.URLDecoder;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -45,12 +43,15 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.base.Joiner;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class renders the various pages that the web app supports.
*/
public class AppController extends Controller implements AMParams {
- private static final Log LOG = LogFactory.getLog(AppController.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AppController.class);
private static final Joiner JOINER = Joiner.on("");
protected final App app;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
index 232fe6b..16686ab 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestLocalContainerLauncher.java
@@ -31,8 +31,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -61,10 +59,12 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestLocalContainerLauncher {
- private static final Log LOG =
- LogFactory.getLog(TestLocalContainerLauncher.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestLocalContainerLauncher.class);
private static File testWorkDir;
private static final String[] localDirs = new String[2];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 47caf44..d951944 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -36,8 +36,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
@@ -87,12 +85,14 @@ import org.mockito.Mockito;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestJobHistoryEventHandler {
- private static final Log LOG = LogFactory
- .getLog(TestJobHistoryEventHandler.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestJobHistoryEventHandler.class);
private static MiniDFSCluster dfsCluster = null;
private static String coreSitePath;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
index b6d8bbf..b81f716 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobSummary.java
@@ -18,19 +18,20 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestJobSummary {
- private static final Log LOG = LogFactory.getLog(TestJobSummary.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestJobSummary.class);
private JobSummary summary = new JobSummary();
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
index 2d0b5ea..a6e57ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
@@ -24,8 +24,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.EnumSet;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -101,6 +99,8 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -109,7 +109,7 @@ import org.junit.Assert;
*/
@SuppressWarnings("unchecked")
public class MRApp extends MRAppMaster {
- private static final Log LOG = LogFactory.getLog(MRApp.class);
+ private static final Logger LOG = LoggerFactory.getLogger(MRApp.class);
/**
* The available resource of each container allocated.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
index 322984e..025a8fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -52,10 +53,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.junit.Test;
+import org.slf4j.event.Level;
public class MRAppBenchmark {
@@ -67,8 +66,7 @@ public class MRAppBenchmark {
* @throws Exception On application failure
*/
public void run(MRApp app) throws Exception {
- Logger rootLogger = LogManager.getRootLogger();
- rootLogger.setLevel(Level.WARN);
+ GenericTestUtils.setRootLogLevel(Level.WARN);
long startTime = System.currentTimeMillis();
Job job = app.submit(new Configuration());
while (!job.getReport().getJobState().equals(JobState.SUCCEEDED)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
index 203958d..fbce003 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
@@ -41,8 +41,6 @@ import java.util.Map;
import org.junit.Assert;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -77,6 +75,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -84,18 +83,19 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import org.slf4j.event.Level;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestMRAppMaster {
- private static final Log LOG = LogFactory.getLog(TestMRAppMaster.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMRAppMaster.class);
private static final Path TEST_ROOT_DIR =
new Path(System.getProperty("test.build.data", "target/test-dir"));
private static final Path testDir = new Path(TEST_ROOT_DIR,
@@ -408,8 +408,7 @@ public class TestMRAppMaster {
@Test
public void testMRAppMasterCredentials() throws Exception {
- Logger rootLogger = LogManager.getRootLogger();
- rootLogger.setLevel(Level.DEBUG);
+ GenericTestUtils.setRootLogLevel(Level.DEBUG);
// Simulate credentials passed to AM via client->RM->NM
Credentials credentials = new Credentials();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
index 6332c5d..17e07b1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java
@@ -38,8 +38,6 @@ import java.util.Map;
import java.util.concurrent.TimeoutException;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
@@ -103,11 +101,13 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings({"unchecked", "rawtypes"})
public class TestRecovery {
- private static final Log LOG = LogFactory.getLog(TestRecovery.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestRecovery.class);
private static Path outputDir = new Path(new File("target",
TestRecovery.class.getName()).getAbsolutePath() +
Path.SEPARATOR + "out");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
index e1fa198..ed70a98 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
@@ -29,8 +29,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskCompletionEvent;
@@ -81,6 +79,8 @@ import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings({"unchecked", "rawtypes"})
public class TestRuntimeEstimators {
@@ -98,7 +98,8 @@ public class TestRuntimeEstimators {
AppContext myAppContext;
- private static final Log LOG = LogFactory.getLog(TestRuntimeEstimators.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRuntimeEstimators.class);
private final AtomicInteger slotsInUse = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
index 57573cc..4a5d480 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java
@@ -24,8 +24,6 @@ import java.util.Map;
import org.apache.hadoop.mapreduce.TaskType;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRConfig;
@@ -40,10 +38,13 @@ import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestMapReduceChildJVM {
- private static final Log LOG = LogFactory.getLog(TestMapReduceChildJVM.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMapReduceChildJVM.class);
@Test (timeout = 30000)
public void testCommandLine() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
index 4abdff8..62d4cc0 100755
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
@@ -28,8 +28,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Task;
@@ -69,11 +67,13 @@ import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("rawtypes")
public class TestTaskImpl {
- private static final Log LOG = LogFactory.getLog(TestTaskImpl.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestTaskImpl.class);
private JobConf conf;
private TaskAttemptListener taskAttemptListener;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
index a93bf88..d5bf03d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
@@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.Server;
@@ -93,6 +91,8 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestContainerLauncher {
@@ -101,7 +101,8 @@ public class TestContainerLauncher {
Configuration conf;
Server server;
- static final Log LOG = LogFactory.getLog(TestContainerLauncher.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestContainerLauncher.class);
@Test (timeout = 10000)
public void testPoolSize() throws InterruptedException {
@@ -446,7 +447,7 @@ public class TestContainerLauncher {
// make the thread sleep to look like its not going to respond
Thread.sleep(15000);
} catch (Exception e) {
- LOG.error(e);
+ LOG.error("Setup thread sleep interrupted: ", e);
throw new UndeclaredThrowableException(e);
}
status.setState(ContainerState.RUNNING);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index 53af631..0ae0380 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -32,8 +32,6 @@ import java.util.Map;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -82,9 +80,12 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestContainerLauncherImpl {
- static final Log LOG = LogFactory.getLog(TestContainerLauncherImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestContainerLauncherImpl.class);
private static final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/453d48bd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index e4a8a1a..85e4181 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -48,8 +48,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -154,12 +152,14 @@ import org.junit.Test;
import com.google.common.base.Supplier;
import org.mockito.InOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("unchecked")
public class TestRMContainerAllocator {
- static final Log LOG = LogFactory
- .getLog(TestRMContainerAllocator.class);
+ static final Logger LOG = LoggerFactory
+ .getLogger(TestRMContainerAllocator.class);
static final RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-7269. Tracking URL in the app
state does not get redirected to ApplicationMaster for Running applications.
Contributed by Wangda Tan
Posted by kk...@apache.org.
YARN-7269. Tracking URL in the app state does not get redirected to ApplicationMaster for Running applications. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4111e6c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4111e6c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4111e6c7
Branch: refs/heads/YARN-6592
Commit: 4111e6c7819d66e17e0932302c589777a355dbae
Parents: 453d48b
Author: Jian He <ji...@apache.org>
Authored: Tue Oct 3 09:54:38 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Tue Oct 3 09:54:38 2017 -0700
----------------------------------------------------------------------
.../webproxy/amfilter/AmFilterInitializer.java | 38 +++++++++--
.../server/webproxy/amfilter/AmIpFilter.java | 67 ++++++++------------
.../server/webproxy/amfilter/TestAmFilter.java | 8 +--
.../amfilter/TestAmFilterInitializer.java | 14 ++--
4 files changed, 70 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index 6841616..fa54c79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -18,21 +18,26 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.http.FilterInitializer;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
public class AmFilterInitializer extends FilterInitializer {
private static final String FILTER_NAME = "AM_PROXY_FILTER";
private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName();
+ public static final String RM_HA_URLS = "RM_HA_URLS";
@Override
public void initFilter(FilterContainer container, Configuration conf) {
@@ -55,6 +60,29 @@ public class AmFilterInitializer extends FilterInitializer {
sb.setLength(sb.length() - 1);
params.put(AmIpFilter.PROXY_URI_BASES, sb.toString());
container.addFilter(FILTER_NAME, FILTER_CLASS, params);
+
+ // Handle RM HA urls
+ List<String> urls = new ArrayList<>();
+
+ // Include yarn-site.xml in the classpath
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ for (String rmId : getRmIds(yarnConf)) {
+ String url = getUrlByRmId(yarnConf, rmId);
+ urls.add(url);
+ }
+ params.put(RM_HA_URLS, StringUtils.join(",", urls));
+ }
+
+ private Collection<String> getRmIds(Configuration conf) {
+ return conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
+ }
+
+ private String getUrlByRmId(Configuration conf, String rmId) {
+ String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ?
+ YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS :
+ YarnConfiguration.RM_WEBAPP_ADDRESS;
+ String host = conf.get(HAUtil.addSuffix(addressPropertyPrefix, rmId));
+ return host;
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
index cdab405..ac4afa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
@@ -18,17 +18,12 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.net.HttpURLConnection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.Collection;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -39,15 +34,16 @@ import javax.servlet.ServletResponse;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
@Public
public class AmIpFilter implements Filter {
@@ -70,6 +66,7 @@ public class AmIpFilter implements Filter {
private long lastUpdate;
@VisibleForTesting
Map<String, String> proxyUriBases;
+ String rmUrls[] = null;
@Override
public void init(FilterConfig conf) throws ServletException {
@@ -95,6 +92,10 @@ public class AmIpFilter implements Filter {
}
}
}
+
+ if (conf.getInitParameter(AmFilterInitializer.RM_HA_URLS) != null) {
+ rmUrls = conf.getInitParameter(AmFilterInitializer.RM_HA_URLS).split(",");
+ }
}
protected Set<String> getProxyAddresses() throws ServletException {
@@ -196,13 +197,11 @@ public class AmIpFilter implements Filter {
if (proxyUriBases.size() == 1) {
// external proxy or not RM HA
addr = proxyUriBases.values().iterator().next();
- } else {
- // RM HA
- YarnConfiguration conf = new YarnConfiguration();
- for (String rmId : getRmIds(conf)) {
- String url = getUrlByRmId(conf, rmId);
- if (isValidUrl(url)) {
- addr = url;
+ } else if (rmUrls != null) {
+ for (String url : rmUrls) {
+ String host = proxyUriBases.get(url);
+ if (isValidUrl(host)) {
+ addr = host;
break;
}
}
@@ -215,20 +214,6 @@ public class AmIpFilter implements Filter {
return addr;
}
- @VisibleForTesting
- Collection<String> getRmIds(YarnConfiguration conf) {
- return conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
- }
-
- @VisibleForTesting
- String getUrlByRmId(YarnConfiguration conf, String rmId) {
- String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ?
- YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS :
- YarnConfiguration.RM_WEBAPP_ADDRESS;
- String host = conf.get(HAUtil.addSuffix(addressPropertyPrefix, rmId));
- return proxyUriBases.get(host);
- }
-
private boolean isValidUrl(String url) {
boolean isValid = false;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
index 687faea..f775c6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
@@ -159,13 +159,7 @@ public class TestAmFilter {
spy.proxyUriBases = new HashMap<>();
spy.proxyUriBases.put(rm1, rm1Url);
spy.proxyUriBases.put(rm2, rm2Url);
-
- Collection<String> rmIds = new ArrayList<>(Arrays.asList(rm1, rm2));
- Mockito.doReturn(rmIds).when(spy).getRmIds(Mockito.any());
- Mockito.doReturn(rm1Url).when(spy)
- .getUrlByRmId(Mockito.any(), Mockito.eq(rm2));
- Mockito.doReturn(rm2Url).when(spy)
- .getUrlByRmId(Mockito.any(), Mockito.eq(rm1));
+ spy.rmUrls = new String[] { rm1, rm2 };
assertEquals(spy.findRedirectUrl(), rm1Url);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4111e6c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
index 07478ca..b621012 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
@@ -56,10 +56,11 @@ public class TestAmFilterInitializer {
AmFilterInitializer afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
assertEquals("host1", con.givenParameters.get(AmIpFilter.PROXY_HOSTS));
assertEquals("http://host1:1000/foo",
con.givenParameters.get(AmIpFilter.PROXY_URI_BASES));
+ assertEquals("", con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
// Check a single RM_WEBAPP_ADDRESS
con = new MockFilterContainer();
@@ -68,10 +69,11 @@ public class TestAmFilterInitializer {
afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
assertEquals("host2", con.givenParameters.get(AmIpFilter.PROXY_HOSTS));
assertEquals("http://host2:2000/foo",
con.givenParameters.get(AmIpFilter.PROXY_URI_BASES));
+ assertEquals("", con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
// Check multiple RM_WEBAPP_ADDRESSes (RM HA)
con = new MockFilterContainer();
@@ -84,7 +86,7 @@ public class TestAmFilterInitializer {
afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
String[] proxyHosts = con.givenParameters.get(AmIpFilter.PROXY_HOSTS)
.split(AmIpFilter.PROXY_HOSTS_DELIMITER);
assertEquals(3, proxyHosts.length);
@@ -99,6 +101,8 @@ public class TestAmFilterInitializer {
assertEquals("http://host2:2000/foo", proxyBases[0]);
assertEquals("http://host3:3000/foo", proxyBases[1]);
assertEquals("http://host4:4000/foo", proxyBases[2]);
+ assertEquals("host2:2000,host3:3000,host4:4000",
+ con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
// Check multiple RM_WEBAPP_ADDRESSes (RM HA) with HTTPS
con = new MockFilterContainer();
@@ -112,7 +116,7 @@ public class TestAmFilterInitializer {
afi = new MockAmFilterInitializer();
assertNull(con.givenParameters);
afi.initFilter(con, conf);
- assertEquals(2, con.givenParameters.size());
+ assertEquals(3, con.givenParameters.size());
proxyHosts = con.givenParameters.get(AmIpFilter.PROXY_HOSTS)
.split(AmIpFilter.PROXY_HOSTS_DELIMITER);
assertEquals(2, proxyHosts.length);
@@ -125,6 +129,8 @@ public class TestAmFilterInitializer {
Arrays.sort(proxyBases);
assertEquals("https://host5:5000/foo", proxyBases[0]);
assertEquals("https://host6:6000/foo", proxyBases[1]);
+ assertEquals("host5:5000,host6:6000",
+ con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDFS-12523. Thread pools in
ErasureCodingWorker do not shutdown. (Huafeng Wang via Lei)
Posted by kk...@apache.org.
HDFS-12523. Thread pools in ErasureCodingWorker do not shutdown. (Huafeng Wang via Lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1267ff22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1267ff22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1267ff22
Branch: refs/heads/YARN-6592
Commit: 1267ff22ce9226b6dd52e3f33cbe3b3094fb0e35
Parents: 9d3e4cc
Author: Lei Xu <le...@apache.org>
Authored: Thu Sep 21 16:10:32 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Tue Sep 26 10:08:16 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 6 +++++-
.../server/datanode/erasurecode/ErasureCodingWorker.java | 11 +++++++++--
.../datanode/erasurecode/StripedReconstructor.java | 8 +++-----
3 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1267ff22/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 042a627..6163d93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1115,7 +1115,7 @@ public class DataNode extends ReconfigurableBase
/**
* Shutdown disk balancer.
*/
- private void shutdownDiskBalancer() {
+ private void shutdownDiskBalancer() {
if (this.diskBalancer != null) {
this.diskBalancer.shutdown();
this.diskBalancer = null;
@@ -2077,6 +2077,10 @@ public class DataNode extends ReconfigurableBase
ipcServer.stop();
}
+ if (ecWorker != null) {
+ ecWorker.shutDown();
+ }
+
if(blockPoolManager != null) {
try {
this.blockPoolManager.shutDownAll(bposArray);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1267ff22/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 07d213c..63498bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.util.Daemon;
import org.slf4j.Logger;
import java.util.Collection;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -149,7 +151,12 @@ public final class ErasureCodingWorker {
return conf;
}
- ThreadPoolExecutor getStripedReadPool() {
- return stripedReadPool;
+ CompletionService<Void> createReadService() {
+ return new ExecutorCompletionService<>(stripedReadPool);
+ }
+
+ public void shutDown() {
+ stripedReconstructionPool.shutdown();
+ stripedReadPool.shutdown();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1267ff22/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index 3202121..bbffcf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -39,8 +39,6 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -110,7 +108,7 @@ abstract class StripedReconstructor {
// position in striped internal block
private long positionInBlock;
private StripedReader stripedReader;
- private ThreadPoolExecutor stripedReadPool;
+ private ErasureCodingWorker erasureCodingWorker;
private final CachingStrategy cachingStrategy;
private long maxTargetLength = 0L;
private final BitSet liveBitSet;
@@ -122,7 +120,7 @@ abstract class StripedReconstructor {
StripedReconstructor(ErasureCodingWorker worker,
StripedReconstructionInfo stripedReconInfo) {
- this.stripedReadPool = worker.getStripedReadPool();
+ this.erasureCodingWorker = worker;
this.datanode = worker.getDatanode();
this.conf = worker.getConf();
this.ecPolicy = stripedReconInfo.getEcPolicy();
@@ -225,7 +223,7 @@ abstract class StripedReconstructor {
}
CompletionService<Void> createReadService() {
- return new ExecutorCompletionService<>(stripedReadPool);
+ return erasureCodingWorker.createReadService();
}
ExtendedBlock getBlockGroup() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by kk...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
new file mode 100644
index 0000000..860320d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -0,0 +1,998 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <ctype.h>
+#include "../modules/common/module-configs.h"
+#include "docker-util.h"
+#include "string-utils.h"
+#include "util.h"
+
+static int read_and_verify_command_file(const char *command_file, const char *docker_command,
+ struct configuration *command_config) {
+ int ret = 0;
+ ret = read_config(command_file, command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (command == NULL || (strcmp(command, docker_command) != 0)) {
+ ret = INCORRECT_COMMAND;
+ }
+ free(command);
+ return ret;
+}
+
+static int add_to_buffer(char *buff, const size_t bufflen, const char *string) {
+ size_t current_len = strlen(buff);
+ size_t string_len = strlen(string);
+ if (current_len + string_len < bufflen - 1) {
+ strncpy(buff + current_len, string, string_len);
+ buff[current_len + string_len] = '\0';
+ return 0;
+ }
+ return -1;
+}
+
+static int add_param_to_command(const struct configuration *command_config, const char *key, const char *param,
+ const int with_argument, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ int ret = 0;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (value != NULL) {
+ if (with_argument) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, value);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ } else if (strcmp(value, "true") == 0) {
+ ret = add_to_buffer(out, outlen, param);
+ }
+ free(value);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ }
+ free(tmp_buffer);
+ return ret;
+}
+
+static int add_param_to_command_if_allowed(const struct configuration *command_config,
+ const struct configuration *executor_cfg,
+ const char *key, const char *allowed_key, const char *param,
+ const int multiple_values, const char prefix,
+ char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *tmp_ptr = NULL;
+ char **values = NULL;
+ char **permitted_values = get_configuration_values_delimiter(allowed_key,
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, executor_cfg,
+ ",");
+ int i = 0, j = 0, permitted = 0, ret = 0;
+ if (multiple_values) {
+ values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ } else {
+ values = (char **) alloc_and_clear_memory(2, sizeof(char *));
+ values[0] = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ values[1] = NULL;
+ if (values[0] == NULL) {
+ ret = 0;
+ goto free_and_exit;
+ }
+ }
+
+ if (values != NULL) {
+ if (permitted_values != NULL) {
+ for (i = 0; values[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ permitted = 0;
+ if(prefix != 0) {
+ tmp_ptr = strchr(values[i], prefix);
+ if (tmp_ptr == NULL) {
+ fprintf(ERRORFILE, "Prefix char '%c' not found in '%s'\n",
+ prefix, values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ for (j = 0; permitted_values[j] != NULL; ++j) {
+ if (prefix == 0) {
+ ret = strcmp(values[i], permitted_values[j]);
+ } else {
+ ret = strncmp(values[i], permitted_values[j], tmp_ptr - values[i]);
+ }
+ if (ret == 0) {
+ permitted = 1;
+ break;
+ }
+ }
+ if (permitted == 1) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, values[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Output buffer too small\n");
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested\n", values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested, "
+ "permitted values list is empty\n", values[0]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free_values(values);
+ free_values(permitted_values);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_docker_config_param(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "docker-config", "--config=", 1, out, outlen);
+}
+
+static int validate_container_name(const char *container_name) {
+ const char *CONTAINER_NAME_PREFIX = "container_";
+ if (0 == strncmp(container_name, CONTAINER_NAME_PREFIX, strlen(CONTAINER_NAME_PREFIX))) {
+ if (1 == validate_container_id(container_name)) {
+ return 0;
+ }
+ }
+ fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", container_name);
+ fflush(ERRORFILE);
+ return INVALID_DOCKER_CONTAINER_NAME;
+}
+
+const char *get_docker_error_message(const int error_code) {
+
+ switch (error_code) {
+ case INVALID_COMMAND_FILE:
+ return "Invalid command file passed";
+ case INCORRECT_COMMAND:
+ return "Incorrect command";
+ case BUFFER_TOO_SMALL:
+ return "Command buffer too small";
+ case INVALID_DOCKER_CONTAINER_NAME:
+ return "Invalid docker container name";
+ case INVALID_DOCKER_IMAGE_NAME:
+ return "Invalid docker image name";
+ case INVALID_DOCKER_USER_NAME:
+ return "Invalid docker user name";
+ case INVALID_DOCKER_INSPECT_FORMAT:
+ return "Invalid docker inspect format";
+ case UNKNOWN_DOCKER_COMMAND:
+ return "Unknown docker command";
+ case INVALID_DOCKER_NETWORK:
+ return "Invalid docker network";
+ case INVALID_DOCKER_CAPABILITY:
+ return "Invalid docker capability";
+ case PRIVILEGED_CONTAINERS_DISABLED:
+ return "Privileged containers are disabled";
+ case INVALID_DOCKER_MOUNT:
+ return "Invalid docker mount";
+ case INVALID_DOCKER_RO_MOUNT:
+ return "Invalid docker read-only mount";
+ case INVALID_DOCKER_RW_MOUNT:
+ return "Invalid docker read-write mount";
+ case MOUNT_ACCESS_ERROR:
+ return "Mount access error";
+ case INVALID_DOCKER_DEVICE:
+ return "Invalid docker device";
+ default:
+ return "Unknown error";
+ }
+}
+
+char *get_docker_binary(const struct configuration *conf) {
+ char *docker_binary = NULL;
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (docker_binary == NULL) {
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, "", conf);
+ if (docker_binary == NULL) {
+ docker_binary = strdup("/usr/bin/docker");
+ }
+ }
+ return docker_binary;
+}
+
+int docker_module_enabled(const struct configuration *conf) {
+ struct section *section = get_configuration_section(CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (section != NULL) {
+ return module_enabled(section, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION);
+ }
+ return 0;
+}
+
+int get_docker_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ struct configuration command_config = {0, NULL};
+ ret = read_config(command_file, &command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (strcmp(DOCKER_INSPECT_COMMAND, command) == 0) {
+ return get_docker_inspect_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_LOAD_COMMAND, command) == 0) {
+ return get_docker_load_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_PULL_COMMAND, command) == 0) {
+ return get_docker_pull_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RM_COMMAND, command) == 0) {
+ return get_docker_rm_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RUN_COMMAND, command) == 0) {
+ return get_docker_run_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_STOP_COMMAND, command) == 0) {
+ return get_docker_stop_command(command_file, conf, out, outlen);
+ } else {
+ return UNKNOWN_DOCKER_COMMAND;
+ }
+}
+
+int get_docker_inspect_command(const char *command_file, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ const char *valid_format_strings[] = { "{{.State.Status}}",
+ "{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}" };
+ int ret = 0, i = 0, valid_format = 0;
+ char *format = NULL, *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_INSPECT_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (format == NULL) {
+ free(container_name);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+ for (i = 0; i < 2; ++i) {
+ if (strcmp(format, valid_format_strings[i]) == 0) {
+ valid_format = 1;
+ break;
+ }
+ }
+ if (valid_format != 1) {
+ fprintf(ERRORFILE, "Invalid format option '%s' not permitted\n", format);
+ free(container_name);
+ free(format);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(container_name);
+ free(format);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_INSPECT_COMMAND);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " --format=");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, format);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, container_name);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ free(format);
+ free(container_name);
+ return 0;
+
+ free_and_exit:
+ free(format);
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_load_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_LOAD_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_LOAD_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --i=", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int validate_docker_image_name(const char *image_name) {
+ const char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
+ return execute_regex_match(regex_str, image_name);
+}
+
+int get_docker_pull_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_PULL_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL || validate_docker_image_name(image_name) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_PULL_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " ", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_rm_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RM_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RM_COMMAND);
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_stop_command(const char *command_file, const struct configuration *conf,
+ char *out, const size_t outlen) {
+ int ret = 0;
+ size_t len = 0, i = 0;
+ char *value = NULL;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_STOP_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_STOP_COMMAND);
+ if (ret == 0) {
+ value = get_configuration_value("time", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (value != NULL) {
+ len = strlen(value);
+ for (i = 0; i < len; ++i) {
+ if (isdigit(value[i]) == 0) {
+ fprintf(ERRORFILE, "Value for time is not a number '%s'\n", value);
+ free(container_name);
+ memset(out, 0, outlen);
+ return INVALID_DOCKER_STOP_COMMAND;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " --time=");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, value);
+ }
+ if (ret != 0) {
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int detach_container(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "detach", "-d ", 0, out, outlen);
+}
+
+static int rm_container_on_exit(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "rm", "--rm ", 0, out, outlen);
+}
+
+static int set_container_workdir(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "workdir", "--workdir=", 1, out, outlen);
+}
+
+static int set_cgroup_parent(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "cgroup-parent", "--cgroup-parent=", 1, out, outlen);
+}
+
+static int set_hostname(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "hostname", "--hostname=", 1, out, outlen);
+}
+
+static int set_group_add(const struct configuration *command_config, char *out, const size_t outlen) {
+ int i = 0, ret = 0;
+ char **group_add = get_configuration_values_delimiter("group-add", DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = NULL;
+
+ if (group_add != NULL) {
+ for (i = 0; group_add[i] != NULL; ++i) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--group-add ", group_add[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ }
+ return ret;
+}
+
+static int set_network(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "net",
+ "docker.allowed.networks", "--net=",
+ 0, 0, out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Could not find requested network in allowed networks\n");
+ ret = INVALID_DOCKER_NETWORK;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_capabilities(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+
+ ret = add_to_buffer(out, outlen, "--cap-drop='ALL' ");
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ ret = add_param_to_command_if_allowed(command_config, conf, "cap-add",
+ "docker.allowed.capabilities",
+ "--cap-add=", 1, 0,
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker capability requested\n");
+ ret = INVALID_DOCKER_CAPABILITY;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_devices(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "devices", "docker.allowed.devices", "--device=", 1, ':',
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker device requested\n");
+ ret = INVALID_DOCKER_DEVICE;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+/**
+ * Helper function to help normalize mounts for checking if mounts are
+ * permitted. The function does the following -
+ * 1. Find the canonical path for mount using realpath
+ * 2. If the path is a directory, add a '/' at the end (if not present)
+ * 3. Return a copy of the canonicalised path(to be freed by the caller)
+ * @param mount path to be canonicalised
+ * @return pointer to canonicalised path, NULL on error
+ */
+static char* normalize_mount(const char* mount) {
+ int ret = 0;
+ struct stat buff;
+ char *ret_ptr = NULL, *real_mount = NULL;
+ if (mount == NULL) {
+ return NULL;
+ }
+ real_mount = realpath(mount, NULL);
+ if (real_mount == NULL) {
+ fprintf(ERRORFILE, "Could not determine real path of mount '%s'\n", mount);
+ free(real_mount);
+ return NULL;
+ }
+ ret = stat(real_mount, &buff);
+ if (ret == 0) {
+ if (S_ISDIR(buff.st_mode)) {
+ size_t len = strlen(real_mount);
+ if (len <= 0) {
+ return NULL;
+ }
+ if (real_mount[len - 1] != '/') {
+ ret_ptr = (char *) alloc_and_clear_memory(len + 2, sizeof(char));
+ strncpy(ret_ptr, real_mount, len);
+ ret_ptr[len] = '/';
+ ret_ptr[len + 1] = '\0';
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ fprintf(ERRORFILE, "Could not stat path '%s'\n", real_mount);
+ ret_ptr = NULL;
+ }
+ free(real_mount);
+ return ret_ptr;
+}
+
+static int normalize_mounts(char **mounts) {
+ int i = 0;
+ char *tmp = NULL;
+ if (mounts == NULL) {
+ return 0;
+ }
+ for (i = 0; mounts[i] != NULL; ++i) {
+ tmp = normalize_mount(mounts[i]);
+ if (tmp == NULL) {
+ return -1;
+ }
+ free(mounts[i]);
+ mounts[i] = tmp;
+ }
+ return 0;
+}
+
+static int check_mount_permitted(const char **permitted_mounts, const char *requested) {
+ int i = 0, ret = 0;
+ size_t permitted_mount_len = 0;
+ char *normalized_path = normalize_mount(requested);
+ if (permitted_mounts == NULL) {
+ return 0;
+ }
+ if (normalized_path == NULL) {
+ return -1;
+ }
+ for (i = 0; permitted_mounts[i] != NULL; ++i) {
+ if (strcmp(normalized_path, permitted_mounts[i]) == 0) {
+ ret = 1;
+ break;
+ }
+ // directory check
+ permitted_mount_len = strlen(permitted_mounts[i]);
+ if (permitted_mount_len > 0
+ && permitted_mounts[i][permitted_mount_len - 1] == '/') {
+ if (strncmp(normalized_path, permitted_mounts[i], permitted_mount_len) == 0) {
+ ret = 1;
+ break;
+ }
+ }
+
+ }
+ free(normalized_path);
+ return ret;
+}
+
+static char* get_mount_source(const char *mount) {
+ char *src_mount = NULL;
+ const char *tmp = NULL;
+ tmp = strchr(mount, ':');
+ if (tmp == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ src_mount = strndup(mount, tmp - mount);
+ return src_mount;
+}
+
+static int add_mounts(const struct configuration *command_config, const struct configuration *conf, const char *key,
+ const int ro, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ const char *ro_suffix = "";
+ const char *tmp_path_buffer[2] = {NULL, NULL};
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char **permitted_ro_mounts = get_configuration_values_delimiter("docker.allowed.ro-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ char *tmp_buffer_2 = NULL, *mount_src = NULL;
+ const char *container_executor_cfg_path = normalize_mount(get_config_path(""));
+ int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
+ if (ro != 0) {
+ ro_suffix = ":ro";
+ }
+
+ if (values != NULL) {
+ ret = normalize_mounts(permitted_ro_mounts);
+ ret |= normalize_mounts(permitted_rw_mounts);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
+ ret = MOUNT_ACCESS_ERROR;
+ goto free_and_exit;
+ }
+ for (i = 0; values[i] != NULL; ++i) {
+ mount_src = get_mount_source(values[i]);
+ if (mount_src == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
+ permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
+ if (permitted_ro == -1 || permitted_rw == -1) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ // rw mount
+ if (ro == 0) {
+ if (permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ } else {
+ // determine if the user can modify the container-executor.cfg file
+ tmp_path_buffer[0] = normalize_mount(mount_src);
+ // just re-use the function, flip the args to check if the container-executor path is in the requested
+ // mount point
+ ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
+ free((void *) tmp_path_buffer[0]);
+ if (ret == 1) {
+ fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
+ values[i]);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ }
+ }
+ }
+ //ro mount
+ if (ro != 0 && permitted_ro == 0 && permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker ro mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RO_MOUNT;
+ goto free_and_exit;
+ }
+ tmp_buffer_2 = (char *) alloc_and_clear_memory(strlen(values[i]) + strlen(ro_suffix) + 1, sizeof(char));
+ strncpy(tmp_buffer_2, values[i], strlen(values[i]));
+ strncpy(tmp_buffer_2 + strlen(values[i]), ro_suffix, strlen(ro_suffix));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "-v ", tmp_buffer_2);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer_2);
+ free(mount_src);
+ tmp_buffer_2 = NULL;
+ mount_src = NULL;
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ }
+ }
+
+ free_and_exit:
+ free_values(permitted_ro_mounts);
+ free_values(permitted_rw_mounts);
+ free_values(values);
+ free(mount_src);
+ free((void *) container_executor_cfg_path);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_ro_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "ro-mounts", 1, out, outlen);
+}
+
+static int add_rw_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
+}
+
+static int set_privileged(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, command_config);
+ char *privileged_container_enabled
+ = get_configuration_value("docker.privileged-containers.enabled", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ int ret = 0;
+
+ if (value != NULL && strcmp(value, "true") == 0) {
+ if (privileged_container_enabled != NULL) {
+ if (strcmp(privileged_container_enabled, "1") == 0) {
+ ret = add_to_buffer(out, outlen, "--privileged ");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free(tmp_buffer);
+ free(value);
+ free(privileged_container_enabled);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+int get_docker_run_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0, i = 0;
+ char *container_name = NULL, *user = NULL, *image = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ char **launch_command = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+ user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (user == NULL) {
+ return INVALID_DOCKER_USER_NAME;
+ }
+ image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image == NULL || validate_docker_image_name(image) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RUN_COMMAND);
+ if(ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --name=", container_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--user=", user);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ ret = detach_container(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = rm_container_on_exit(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_container_workdir(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_network(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_ro_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_rw_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_cgroup_parent(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_privileged(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_capabilities(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_hostname(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_group_add(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_devices(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", image);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
+ ",");
+ if (launch_command != NULL) {
+ for (i = 0; launch_command[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", launch_command[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ free_values(launch_command);
+ free(tmp_buffer);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ free_values(launch_command);
+ }
+ free(tmp_buffer);
+ return 0;
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
new file mode 100644
index 0000000..37ec880
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+
+#include "configuration.h"
+
+#define CONTAINER_EXECUTOR_CFG_DOCKER_SECTION "docker"
+#define DOCKER_BINARY_KEY "docker.binary"
+#define DOCKER_COMMAND_FILE_SECTION "docker-command-execution"
+#define DOCKER_INSPECT_COMMAND "inspect"
+#define DOCKER_LOAD_COMMAND "load"
+#define DOCKER_PULL_COMMAND "pull"
+#define DOCKER_RM_COMMAND "rm"
+#define DOCKER_RUN_COMMAND "run"
+#define DOCKER_STOP_COMMAND "stop"
+
+
+enum docker_error_codes {
+ INVALID_COMMAND_FILE = 1,
+ INCORRECT_COMMAND,
+ BUFFER_TOO_SMALL,
+ INVALID_DOCKER_CONTAINER_NAME,
+ INVALID_DOCKER_IMAGE_NAME,
+ INVALID_DOCKER_USER_NAME,
+ INVALID_DOCKER_INSPECT_FORMAT,
+ UNKNOWN_DOCKER_COMMAND,
+ INVALID_DOCKER_NETWORK,
+ INVALID_DOCKER_CAPABILITY,
+ PRIVILEGED_CONTAINERS_DISABLED,
+ INVALID_DOCKER_MOUNT,
+ INVALID_DOCKER_RO_MOUNT,
+ INVALID_DOCKER_RW_MOUNT,
+ MOUNT_ACCESS_ERROR,
+ INVALID_DOCKER_DEVICE,
+ INVALID_DOCKER_STOP_COMMAND
+};
+
+/**
+ * Get the full path for the docker binary.
+ * @param conf Configuration for the container-executor
+ * @return String containing the docker binary to be freed by the user.
+ */
+char *get_docker_binary(const struct configuration *conf);
+
+/**
+ * Get the Docker command line string. The function will inspect the params file to determine the command to be run.
+ * @param command_file File containing the params for the Docker command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the Docker command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker inspect command line string. The function will verify that the params file is meant for the
+ * inspect command.
+ * @param command_file File containing the params for the Docker inspect command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the inspect command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_inspect_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker load command line string. The function will verify that the params file is meant for the load command.
+ * @param command_file File containing the params for the Docker load command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the load command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_load_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker pull command line string. The function will verify that the params file is meant for the pull command.
+ * @param command_file File containing the params for the Docker pull command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the pull command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_pull_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker rm command line string. The function will verify that the params file is meant for the rm command.
+ * @param command_file File containing the params for the Docker rm command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the rm command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_rm_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker run command line string. The function will verify that the params file is meant for the run command.
+ * @param command_file File containing the params for the Docker run command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the run command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_run_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker stop command line string. The function will verify that the params file is meant for the stop command.
+ * @param command_file File containing the params for the Docker stop command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the stop command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_stop_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Give an error message for the supplied error code
+ * @param error_code the error code
+ * @return const string containing the error message
+ */
+const char *get_docker_error_message(const int error_code);
+
+/**
+ * Determine if the docker module is enabled in the config.
+ * @param conf Configuration structure
+ * @return 1 if enabled, 0 otherwise
+ */
+int docker_module_enabled(const struct configuration *conf);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
index d19c084..40b2c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
@@ -21,7 +21,6 @@
#include <errno.h>
#include <strings.h>
#include <string.h>
-#include <stdio.h>
#include <stdlib.h>
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
new file mode 100644
index 0000000..339bec9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[docker]
+privieged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 64ee717..609dd2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -19,6 +19,7 @@
#include "container-executor.h"
#include "utils/string-utils.h"
#include "util.h"
+#include "get_executable.h"
#include "test/test-container-executor-common.h"
#include <inttypes.h>
@@ -1155,145 +1156,6 @@ void test_trim_function() {
free(trimmed);
}
-void test_sanitize_docker_command() {
-
- char *input[] = {
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run ''''''''",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
- char *expected_output[] = {
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
-
- int input_size = sizeof(input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < input_size; i++) {
- char *command = (char *) calloc(strlen(input[i]) + 1 , sizeof(char));
- strncpy(command, input[i], strlen(input[i]));
- char *op = sanitize_docker_command(command);
- if(strncmp(expected_output[i], op, strlen(expected_output[i])) != 0) {
- printf("FAIL: expected output %s does not match actual output '%s'\n", expected_output[i], op);
- exit(1);
- }
- free(command);
- }
-}
-
-void test_validate_docker_image_name() {
-
- char *good_input[] = {
- "ubuntu",
- "ubuntu:latest",
- "ubuntu:14.04",
- "ubuntu:LATEST",
- "registry.com:5000/user/ubuntu",
- "registry.com:5000/user/ubuntu:latest",
- "registry.com:5000/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu",
- "registry.com/user/ubuntu:latest",
- "registry.com/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu:test-image",
- "registry.com/user/ubuntu:test_image",
- "registry.com/ubuntu",
- "user/ubuntu",
- "user/ubuntu:0.1.2.3",
- "user/ubuntu:latest",
- "user/ubuntu:test_image",
- "user/ubuntu.test:test_image",
- "user/ubuntu-test:test-image",
- "registry.com/ubuntu/ubuntu/ubuntu"
- };
-
- char *bad_input[] = {
- "UBUNTU",
- "registry.com|5000/user/ubuntu",
- "registry.com | 5000/user/ubuntu",
- "ubuntu' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''",
- "bad_host_name:5000/user/ubuntu",
- "registry.com:foo/ubuntu/ubuntu/ubuntu",
- "registry.com/ubuntu:foo/ubuntu/ubuntu"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_docker_image_name(good_input[i]);
- if(0 != op) {
- printf("\nFAIL: docker image name %s is invalid", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_docker_image_name(bad_input[j]);
- if(1 != op) {
- printf("\nFAIL: docker image name %s is valid, expected invalid", bad_input[j]);
- exit(1);
- }
- }
-}
-
-void test_validate_container_id() {
- char *good_input[] = {
- "container_e134_1499953498516_50875_01_000007",
- "container_1499953498516_50875_01_000007",
- "container_e1_12312_11111_02_000001"
- };
-
- char *bad_input[] = {
- "CONTAINER",
- "container_e1_12312_11111_02_000001 | /tmp/file"
- "container_e1_12312_11111_02_000001 || # /tmp/file",
- "container_e1_12312_11111_02_000001 # /tmp/file",
- "container_e1_12312_11111_02_000001' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_container_id(good_input[i]);
- if(1 != op) {
- printf("FAIL: docker container name %s is invalid\n", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_container_id(bad_input[j]);
- if(0 != op) {
- printf("FAIL: docker container name %s is valid, expected invalid\n", bad_input[j]);
- exit(1);
- }
- }
-}
-
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1388,15 +1250,6 @@ int main(int argc, char **argv) {
printf("\nTesting is_feature_enabled()\n");
test_is_feature_enabled();
- printf("\nTesting sanitize docker commands()\n");
- test_sanitize_docker_command();
-
- printf("\nTesting validate_docker_image_name()\n");
- test_validate_docker_image_name();
-
- printf("\nTesting validate_container_id()\n");
- test_validate_container_id();
-
test_check_user(0);
#ifdef __APPLE__
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
index 2ec7b2a..b96dea1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
@@ -17,7 +17,7 @@
*/
#include <gtest/gtest.h>
-#include <sstream>
+#include <vector>
extern "C" {
#include "util.h"
@@ -135,4 +135,39 @@ namespace ContainerExecutor {
ASSERT_STREQ("foo", trimmed);
free(trimmed);
}
+
+ TEST_F(TestUtil, test_escape_single_quote) {
+ std::vector<std::pair<std::string, std::string> > input_output_vec;
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'abcd'", "'\"'\"'abcd'\"'\"'"));
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'", "'\"'\"'"));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = input_output_vec.begin(); itr != input_output_vec.end(); ++itr) {
+ char *ret = escape_single_quote(itr->first.c_str());
+ ASSERT_STREQ(itr->second.c_str(), ret);
+ free(ret);
+ }
+ }
+
+ TEST_F(TestUtil, test_quote_and_append_arg) {
+
+ char *tmp = static_cast<char *>(malloc(4096));
+ size_t tmp_size = 4096;
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "ab'cd");
+ ASSERT_STREQ("param='ab'\"'\"'cd' ", tmp);
+ free(tmp);
+
+ tmp = static_cast<char *>(malloc(4));
+ tmp_size = 4;
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+ ASSERT_EQ(1040, tmp_size);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
index 037816a..b259c6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
@@ -88,6 +88,39 @@
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
-}
+ }
+
+ TEST_F(TestStringUtils, test_validate_container_id) {
+
+ const char *good_input[] = {
+ "container_e134_1499953498516_50875_01_000007",
+ "container_1499953498516_50875_01_000007",
+ "container_e1_12312_11111_02_000001"
+ };
+
+ const char *bad_input[] = {
+ "CONTAINER",
+ "container_e1_12312_11111_02_000001 | /tmp/file"
+ "container_e1_12312_11111_02_000001 || # /tmp/file",
+ "container_e1_12312_11111_02_000001 # /tmp/file",
+ "container_e1_12312_11111_02_000001' || touch /tmp/file #",
+ "ubuntu || touch /tmp/file #",
+ "''''''''"
+ };
+
+ int good_input_size = sizeof(good_input) / sizeof(char *);
+ int i = 0;
+ for (i = 0; i < good_input_size; i++) {
+ int op = validate_container_id(good_input[i]);
+ ASSERT_EQ(1, op);
+ }
+
+ int bad_input_size = sizeof(bad_input) / sizeof(char *);
+ int j = 0;
+ for (j = 0; j < bad_input_size; j++) {
+ int op = validate_container_id(bad_input[j]);
+ ASSERT_EQ(0, op);
+ }
+ }
-} // namespace ContainerExecutor
\ No newline at end of file
+} // namespace ContainerExecutor
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HADOOP-14902.
LoadGenerator#genFile write close timing is incorrectly calculated.
Contributed by Hanisha Koneru
Posted by kk...@apache.org.
HADOOP-14902. LoadGenerator#genFile write close timing is incorrectly calculated. Contributed by Hanisha Koneru
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f789fe0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f789fe0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f789fe0
Branch: refs/heads/YARN-6592
Commit: 6f789fe05766a61b12ca10df3f26ee354eac84aa
Parents: ca669f9
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 28 16:38:30 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 28 16:38:30 2017 -0500
----------------------------------------------------------------------
.../apache/hadoop/fs/loadGenerator/LoadGenerator.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f789fe0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
index 0bb1b46..b74e75d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
@@ -322,6 +322,7 @@ public class LoadGenerator extends Configured implements Tool {
private void genFile(Path file, long fileSize) throws IOException {
long startTimestamp = Time.monotonicNow();
FSDataOutputStream out = null;
+ boolean isOutClosed = false;
try {
out = fc.create(file,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
@@ -337,11 +338,15 @@ public class LoadGenerator extends Configured implements Tool {
i -= s;
}
- startTimestamp = Time.monotonicNow();
- executionTime[WRITE_CLOSE] += (Time.monotonicNow() - startTimestamp);
+ startTime = Time.monotonicNow();
+ out.close();
+ executionTime[WRITE_CLOSE] += (Time.monotonicNow() - startTime);
numOfOps[WRITE_CLOSE]++;
+ isOutClosed = true;
} finally {
- IOUtils.cleanupWithLogger(LOG, out);
+ if (!isOutClosed && out != null) {
+ out.close();
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: MAPREDUCE-6968. Staging directory
erasure coding config property has a typo. Contributed by Jason Lowe
Posted by kk...@apache.org.
MAPREDUCE-6968. Staging directory erasure coding config property has a typo. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9df05005
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9df05005
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9df05005
Branch: refs/heads/YARN-6592
Commit: 9df05005ac258441eeceec25ecef7b28f9f50a02
Parents: 1267ff2
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 26 14:52:27 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 26 14:52:27 2017 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9df05005/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 86abb42..cf59730 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1039,7 +1039,7 @@ public interface MRJobConfig {
boolean DEFAULT_FINISH_JOB_WHEN_REDUCERS_DONE = true;
String MR_AM_STAGING_DIR_ERASURECODING_ENABLED =
- MR_AM_STAGING_DIR + "erasurecoding.enabled";
+ MR_AM_STAGING_DIR + ".erasurecoding.enabled";
boolean DEFAULT_MR_AM_STAGING_ERASURECODING_ENABLED = false;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HADOOP-14893. WritableRpcEngine
should use Time.monotonicNow. Contributed by Chetna Chaudhari.
Posted by kk...@apache.org.
HADOOP-14893. WritableRpcEngine should use Time.monotonicNow. Contributed by Chetna Chaudhari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d08b8c80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d08b8c80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d08b8c80
Branch: refs/heads/YARN-6592
Commit: d08b8c801a908b4242e7b21a54f3b1e4072f1eae
Parents: a2b31e3
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Sep 26 16:45:39 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Sep 26 16:46:03 2017 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d08b8c80/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
index fa0726d..f2b5862 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
@@ -231,7 +231,7 @@ public class WritableRpcEngine implements RpcEngine {
throws Throwable {
long startTime = 0;
if (LOG.isDebugEnabled()) {
- startTime = Time.now();
+ startTime = Time.monotonicNow();
}
// if Tracing is on then start a new span for this rpc.
@@ -251,7 +251,7 @@ public class WritableRpcEngine implements RpcEngine {
if (traceScope != null) traceScope.close();
}
if (LOG.isDebugEnabled()) {
- long callTime = Time.now() - startTime;
+ long callTime = Time.monotonicNow() - startTime;
LOG.debug("Call: " + method.getName() + " " + callTime);
}
return value.get();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-65. Reduce RM app memory
footprint once app has completed. Contributed by Manikandan R.
Posted by kk...@apache.org.
YARN-65. Reduce RM app memory footprint once app has completed. Contributed by Manikandan R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06e5a7b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06e5a7b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06e5a7b5
Branch: refs/heads/YARN-6592
Commit: 06e5a7b5cf141420d3a411088b87acba72e68cad
Parents: d08b8c8
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Sep 26 14:33:48 2017 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Sep 26 14:34:09 2017 +0530
----------------------------------------------------------------------
.../server/resourcemanager/rmapp/RMAppImpl.java | 11 +-
.../resourcemanager/MockMemoryRMStateStore.java | 130 ++++++++++++
.../yarn/server/resourcemanager/MockRM.java | 2 +-
.../resourcemanager/MockRMMemoryStateStore.java | 32 ---
.../resourcemanager/TestApplicationCleanup.java | 4 +-
.../TestContainerResourceUsage.java | 4 +-
.../yarn/server/resourcemanager/TestRMHA.java | 4 +-
.../server/resourcemanager/TestRMRestart.java | 47 +++--
.../TestWorkPreservingRMRestart.java | 51 +++--
.../applicationsmanager/TestAMRestart.java | 13 +-
.../rmapp/TestRMAppTransitions.java | 199 +++++++++++++++++--
.../security/TestRMDelegationTokens.java | 6 +-
12 files changed, 414 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 98192ca..09381f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1459,7 +1459,7 @@ public class RMAppImpl implements RMApp, Recoverable {
app.rmContext.getSystemMetricsPublisher()
.appFinished(app, finalState, app.finishTime);
// set the memory free
- app.submissionContext.getAMContainerSpec().setTokensConf(null);
+ app.clearUnusedFields();
};
}
@@ -2021,4 +2021,13 @@ public class RMAppImpl implements RMApp, Recoverable {
public void setApplicationPriority(Priority applicationPriority) {
this.applicationPriority = applicationPriority;
}
+
+ /**
+ * Clear Unused fields to free memory.
+ * @param app
+ */
+ private void clearUnusedFields() {
+ this.submissionContext.setAMContainerSpec(null);
+ this.submissionContext.setLogAggregationContext(null);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
new file mode 100644
index 0000000..698f1c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Test helper for MemoryRMStateStore will make sure the event.
+ */
+public class MockMemoryRMStateStore extends MemoryRMStateStore {
+
+ private Map<ApplicationId, ApplicationSubmissionContext> appSubCtxtCopy =
+ new HashMap<ApplicationId, ApplicationSubmissionContext>();
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ protected EventHandler getRMStateStoreEventHandler() {
+ return rmStateStoreEventHandler;
+ }
+
+ @Override
+ public synchronized RMState loadState() throws Exception {
+
+ RMState cloneState = super.loadState();
+
+ for(Entry<ApplicationId, ApplicationStateData> state :
+ cloneState.getApplicationState().entrySet()) {
+ ApplicationStateData oldStateData = state.getValue();
+ oldStateData.setApplicationSubmissionContext(
+ this.appSubCtxtCopy.get(state.getKey()));
+ cloneState.getApplicationState().put(state.getKey(), oldStateData);
+ }
+ return cloneState;
+ }
+
+ @Override
+ public synchronized void storeApplicationStateInternal(
+ ApplicationId appId, ApplicationStateData appState)
+ throws Exception {
+ // Clone Application Submission Context
+ this.cloneAppSubmissionContext(appState);
+ super.storeApplicationStateInternal(appId, appState);
+ }
+
+ @Override
+ public synchronized void updateApplicationStateInternal(
+ ApplicationId appId, ApplicationStateData appState)
+ throws Exception {
+ // Clone Application Submission Context
+ this.cloneAppSubmissionContext(appState);
+ super.updateApplicationStateInternal(appId, appState);
+ }
+
+ /**
+ * Clone Application Submission Context and Store in Map for
+ * later use.
+ *
+ * @param appState
+ */
+ private void cloneAppSubmissionContext(ApplicationStateData appState) {
+ ApplicationSubmissionContext oldAppSubCtxt =
+ appState.getApplicationSubmissionContext();
+ ApplicationSubmissionContext context =
+ ApplicationSubmissionContext.newInstance(
+ oldAppSubCtxt.getApplicationId(),
+ oldAppSubCtxt.getApplicationName(),
+ oldAppSubCtxt.getQueue(),
+ oldAppSubCtxt.getPriority(),
+ oldAppSubCtxt.getAMContainerSpec(),
+ oldAppSubCtxt.getUnmanagedAM(),
+ oldAppSubCtxt.getCancelTokensWhenComplete(),
+ oldAppSubCtxt.getMaxAppAttempts(),
+ oldAppSubCtxt.getResource()
+ );
+ context.setAttemptFailuresValidityInterval(
+ oldAppSubCtxt.getAttemptFailuresValidityInterval());
+ context.setKeepContainersAcrossApplicationAttempts(
+ oldAppSubCtxt.getKeepContainersAcrossApplicationAttempts());
+ context.setAMContainerResourceRequests(
+ oldAppSubCtxt.getAMContainerResourceRequests());
+ context.setLogAggregationContext(oldAppSubCtxt.getLogAggregationContext());
+ context.setApplicationType(oldAppSubCtxt.getApplicationType());
+ this.appSubCtxtCopy.put(oldAppSubCtxt.getApplicationId(), context);
+ }
+
+ /**
+ * Traverse each app state and replace cloned app sub context
+ * into the state.
+ *
+ * @param actualState
+ * @return actualState
+ */
+ @VisibleForTesting
+ public RMState reloadStateWithClonedAppSubCtxt(RMState actualState) {
+ for(Entry<ApplicationId, ApplicationStateData> state :
+ actualState.getApplicationState().entrySet()) {
+ ApplicationStateData oldStateData = state.getValue();
+ oldStateData.setApplicationSubmissionContext(
+ this.appSubCtxtCopy.get(state.getKey()));
+ actualState.getApplicationState().put(state.getKey(),
+ oldStateData);
+ }
+ return actualState;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index b772e80..973f487 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -164,7 +164,7 @@ public class MockRM extends ResourceManager {
} else {
Class storeClass = getRMContext().getStateStore().getClass();
if (storeClass.equals(MemoryRMStateStore.class)) {
- MockRMMemoryStateStore mockStateStore = new MockRMMemoryStateStore();
+ MockMemoryRMStateStore mockStateStore = new MockMemoryRMStateStore();
mockStateStore.init(conf);
setRMStateStore(mockStateStore);
} else if (storeClass.equals(NullRMStateStore.class)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMMemoryStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMMemoryStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMMemoryStateStore.java
deleted file mode 100644
index d88ee1e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMMemoryStateStore.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager;
-
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
-
-/**
- * Test helper for MemoryRMStateStore will make sure the event.
- */
-public class MockRMMemoryStateStore extends MemoryRMStateStore {
- @SuppressWarnings("rawtypes")
- @Override
- protected EventHandler getRMStateStoreEventHandler() {
- return rmStateStoreEventHandler;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
index ebca7a3..c12ae33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
@@ -293,6 +293,8 @@ public class TestApplicationCleanup {
// start RM
MockRM rm1 = new MockRM(conf);
rm1.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
@@ -304,7 +306,7 @@ public class TestApplicationCleanup {
rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED);
// start new RM
- MockRM rm2 = new MockRM(conf, rm1.getRMStateStore());
+ MockRM rm2 = new MockRM(conf, memStore);
rm2.start();
// nm1 register to rm2, and do a heartbeat
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index 9ed4978..3508ab4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -143,6 +143,8 @@ public class TestContainerResourceUsage {
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
MockRM rm0 = new MockRM(conf);
rm0.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm0.getRMStateStore();
MockNM nm =
new MockNM("127.0.0.1:1234", 65536, rm0.getResourceTrackerService());
nm.registerNode();
@@ -229,7 +231,7 @@ public class TestContainerResourceUsage {
vcoreSeconds, metricsBefore.getVcoreSeconds());
// create new RM to represent RM restart. Load up the state store.
- MockRM rm1 = new MockRM(conf, rm0.getRMStateStore());
+ MockRM rm1 = new MockRM(conf, memStore);
rm1.start();
RMApp app0After =
rm1.getRMContext().getRMApps().get(app0.getApplicationId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
index b5293a5..20e9ff4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
@@ -415,7 +415,7 @@ public class TestRMHA {
configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
Configuration conf = new YarnConfiguration(configuration);
- MemoryRMStateStore memStore = new MockRMMemoryStateStore() {
+ MemoryRMStateStore memStore = new MockMemoryRMStateStore() {
int count = 0;
@Override
@@ -465,7 +465,7 @@ public class TestRMHA {
configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
Configuration conf = new YarnConfiguration(configuration);
- MemoryRMStateStore memStore = new MockRMMemoryStateStore() {
+ MemoryRMStateStore memStore = new MockMemoryRMStateStore() {
@Override
public void updateApplicationState(ApplicationStateData appState) {
notifyStoreOperationFailed(new StoreFencedException());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 0346f4f..f0f51f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -199,7 +199,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
// PHASE 1: create RM and get state
MockRM rm1 = createMockRM(conf);
- MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
Map<ApplicationId, ApplicationStateData> rmAppState =
memStore.getState().getApplicationState();
@@ -679,7 +680,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
@Test (timeout = 60000)
public void testRMRestartWaitForPreviousSucceededAttempt() throws Exception {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
- MemoryRMStateStore memStore = new MockRMMemoryStateStore() {
+ MemoryRMStateStore memStore = new MockMemoryRMStateStore() {
int count = 0;
@Override
@@ -734,7 +735,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
// create RM
MockRM rm1 = createMockRM(conf);
- MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
Map<ApplicationId, ApplicationStateData> rmAppState =
memStore.getState().getApplicationState();
// start RM
@@ -780,7 +782,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
// create RM
MockRM rm1 = createMockRM(conf);
- MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
Map<ApplicationId, ApplicationStateData> rmAppState =
memStore.getState().getApplicationState();
// start RM
@@ -824,18 +827,18 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
@Test (timeout = 60000)
public void testRMRestartKilledAppWithNoAttempts() throws Exception {
- MemoryRMStateStore memStore = new MockRMMemoryStateStore() {
+ MockMemoryRMStateStore memStore = new MockMemoryRMStateStore() {
@Override
public synchronized void storeApplicationAttemptStateInternal(
- ApplicationAttemptId attemptId,
- ApplicationAttemptStateData attemptStateData) throws Exception {
+ ApplicationAttemptId appAttemptId,
+ ApplicationAttemptStateData attemptState) throws Exception {
// ignore attempt saving request.
}
@Override
public synchronized void updateApplicationAttemptStateInternal(
- ApplicationAttemptId attemptId,
- ApplicationAttemptStateData attemptStateData) throws Exception {
+ ApplicationAttemptId appAttemptId,
+ ApplicationAttemptStateData attemptState) throws Exception {
// ignore attempt saving request.
}
};
@@ -868,7 +871,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
// PHASE 1: create RM and get state
MockRM rm1 = createMockRM(conf);
- MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
Map<ApplicationId, ApplicationStateData> rmAppState =
memStore.getState().getApplicationState();
@@ -926,6 +930,9 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
+
// a succeeded app.
RMApp app0 = rm1.submitApp(200, "name", "user", null,
false, "default", 1, null, "myType");
@@ -953,7 +960,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
.appCreated(any(RMApp.class), anyLong());
// restart rm
- MockRM rm2 = new MockRM(conf, rm1.getRMStateStore()) {
+ MockRM rm2 = new MockRM(conf, memStore) {
@Override
protected RMAppManager createRMAppManager() {
return spy(super.createRMAppManager());
@@ -1625,7 +1632,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
// start RM
MockRM rm1 = createMockRM(conf);
rm1.start();
- MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
RMState rmState = memStore.getState();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
@@ -1664,7 +1672,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
// This is to test RM does not get hang on shutdown.
@Test (timeout = 10000)
public void testRMShutdown() throws Exception {
- MemoryRMStateStore memStore = new MockRMMemoryStateStore() {
+ MemoryRMStateStore memStore = new MockMemoryRMStateStore() {
@Override
public synchronized void checkVersion()
throws Exception {
@@ -1743,7 +1751,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
}
};
rm1.start();
- MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
RMApp app1 = null;
try {
app1 = rm1.submitApp(200, "name", "user",
@@ -1767,7 +1776,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
@Test (timeout = 20000)
public void testAppRecoveredInOrderOnRMRestart() throws Exception {
- MemoryRMStateStore memStore = new MockRMMemoryStateStore();
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
for (int i = 10; i > 0; i--) {
@@ -2405,6 +2414,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
MockRM rm1 = new MockRM(conf);
rm1.start();
CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
@@ -2441,7 +2452,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
MockRM rm2 = null;
// start RM2
try {
- rm2 = new MockRM(conf, rm1.getRMStateStore());
+ rm2 = new MockRM(conf, memStore);
rm2.start();
Assert.assertTrue("RM start successfully", true);
} catch (Exception e) {
@@ -2542,6 +2553,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
}
};
rm1.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
// add node label "x" and set node to label mapping
Set<String> clusterNodeLabels = new HashSet<String>();
@@ -2568,7 +2581,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false);
MockRM rm2 = new MockRM(
TestUtils.getConfigurationWithDefaultQueueLabels(conf),
- rm1.getRMStateStore()) {
+ memStore) {
@Override
protected RMNodeLabelsManager createNodeLabelManager() {
RMNodeLabelsManager mgr = new RMNodeLabelsManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index a13cae7..c1cb4c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -572,6 +572,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "");
rm1 = new MockRM(conf);
rm1.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
nm1.registerNode();
@@ -583,7 +585,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{QUEUE_DOESNT_EXIST});
final String noQueue = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_DOESNT_EXIST;
csConf.setCapacity(noQueue, 100);
- rm2 = new MockRM(csConf, rm1.getRMStateStore());
+ rm2 = new MockRM(csConf, memStore);
rm2.start();
UserGroupInformation user2 = UserGroupInformation.createRemoteUser("user2");
@@ -721,11 +723,15 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
private void verifyAppRecoveryWithWrongQueueConfig(
CapacitySchedulerConfiguration csConf, RMApp app, String diagnostics,
- MemoryRMStateStore memStore, RMState state) throws Exception {
+ MockMemoryRMStateStore memStore, RMState state) throws Exception {
// Restart RM with fail-fast as false. App should be killed.
csConf.setBoolean(YarnConfiguration.RM_FAIL_FAST, false);
rm2 = new MockRM(csConf, memStore);
rm2.start();
+
+ MockMemoryRMStateStore memStore2 =
+ (MockMemoryRMStateStore) rm2.getRMStateStore();
+
// Wait for app to be killed.
rm2.waitForState(app.getApplicationId(), RMAppState.KILLED);
ApplicationReport report = rm2.getApplicationReport(app.getApplicationId());
@@ -734,24 +740,27 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
assertEquals(report.getYarnApplicationState(), YarnApplicationState.KILLED);
assertEquals(report.getDiagnostics(), diagnostics);
+ //Reload previous state with cloned app sub context object
+ RMState newState = memStore2.reloadStateWithClonedAppSubCtxt(state);
+
// Remove updated app info(app being KILLED) from state store and reinstate
// state store to previous state i.e. which indicates app is RUNNING.
// This is to simulate app recovery with fail fast config as true.
for(Map.Entry<ApplicationId, ApplicationStateData> entry :
- state.getApplicationState().entrySet()) {
+ newState.getApplicationState().entrySet()) {
ApplicationStateData appState = mock(ApplicationStateData.class);
ApplicationSubmissionContext ctxt =
mock(ApplicationSubmissionContext.class);
when(appState.getApplicationSubmissionContext()).thenReturn(ctxt);
when(ctxt.getApplicationId()).thenReturn(entry.getKey());
- memStore.removeApplicationStateInternal(appState);
- memStore.storeApplicationStateInternal(
+ memStore2.removeApplicationStateInternal(appState);
+ memStore2.storeApplicationStateInternal(
entry.getKey(), entry.getValue());
}
// Now restart RM with fail-fast as true. QueueException should be thrown.
csConf.setBoolean(YarnConfiguration.RM_FAIL_FAST, true);
- MockRM rm = new MockRM(csConf, memStore);
+ MockRM rm = new MockRM(csConf, memStore2);
try {
rm.start();
Assert.fail("QueueException must have been thrown");
@@ -781,6 +790,9 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
setupQueueConfiguration(csConf);
rm1 = new MockRM(csConf);
rm1.start();
+
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm =
new MockNM("127.1.1.1:4321", 8192, rm1.getResourceTrackerService());
nm.registerNode();
@@ -801,7 +813,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
String diags = "Application killed on recovery as it was submitted to " +
"queue QueueB which is no longer a leaf queue after restart.";
verifyAppRecoveryWithWrongQueueConfig(csConf, app, diags,
- (MemoryRMStateStore) rm1.getRMStateStore(), state);
+ memStore, state);
}
//Test behavior of an app if queue is removed during recovery. Test case does
@@ -826,6 +838,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
setupQueueConfiguration(csConf);
rm1 = new MockRM(csConf);
rm1.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
MockNM nm2 =
@@ -853,7 +867,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
rm1.clearQueueMetrics(app2);
// Take a copy of state store so that it can be reset to this state.
- RMState state = rm1.getRMStateStore().loadState();
+ RMState state = memStore.loadState();
// Set new configuration with QueueB removed.
csConf = new CapacitySchedulerConfiguration(conf);
@@ -862,7 +876,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
String diags = "Application killed on recovery as it was submitted to " +
"queue QueueB which no longer exists after restart.";
verifyAppRecoveryWithWrongQueueConfig(csConf, app2, diags,
- (MemoryRMStateStore) rm1.getRMStateStore(), state);
+ memStore, state);
}
private void checkParentQueue(ParentQueue parentQueue, int numContainers,
@@ -931,6 +945,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
public void testContainersNotRecoveredForCompletedApps() throws Exception {
rm1 = new MockRM(conf);
rm1.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
nm1.registerNode();
@@ -938,7 +954,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);
- rm2 = new MockRM(conf, rm1.getRMStateStore());
+ rm2 = new MockRM(conf, memStore);
rm2.start();
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
NMContainerStatus runningContainer =
@@ -1212,6 +1228,9 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
// start RM
rm1 = new MockRM(conf);
rm1.start();
+
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
@@ -1230,7 +1249,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
// start new RM
- rm2 = new MockRM(conf, rm1.getRMStateStore());
+ rm2 = new MockRM(conf, memStore);
rm2.start();
am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
@@ -1370,7 +1389,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
// RM should start correctly.
@Test (timeout = 20000)
public void testAppStateSavedButAttemptStateNotSaved() throws Exception {
- MemoryRMStateStore memStore = new MemoryRMStateStore() {
+ MockMemoryRMStateStore memStore = new MockMemoryRMStateStore() {
@Override public synchronized void updateApplicationAttemptStateInternal(
ApplicationAttemptId appAttemptId,
ApplicationAttemptStateData attemptState) {
@@ -1414,6 +1433,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
// start RM
rm1 = new MockRM(conf);
rm1.start();
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
@@ -1438,8 +1459,10 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
}
// start new RM
- rm2 = new MockRM(conf, rm1.getRMStateStore());
+ rm2 = new MockRM(conf, memStore);
rm2.start();
+ MockMemoryRMStateStore memStore2 =
+ (MockMemoryRMStateStore) rm2.getRMStateStore();
rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED);
rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED);
@@ -1488,7 +1511,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
recoveredApp.getFinalApplicationStatus());
// Restart RM once more to check UAM is not re-run
- MockRM rm3 = new MockRM(conf, rm1.getRMStateStore());
+ MockRM rm3 = new MockRM(conf, memStore2);
rm3.start();
recoveredApp = rm3.getRMContext().getRMApps().get(app0.getApplicationId());
Assert.assertEquals(RMAppState.FINISHED, recoveredApp.getState());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index 528afac..9d0d879 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockMemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -702,8 +703,11 @@ public class TestAMRestart {
// explicitly set max-am-retry count as 2.
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
MockRM rm1 = new MockRM(conf);
- MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
rm1.start();
+
+ MockMemoryRMStateStore memStore =
+ (MockMemoryRMStateStore) rm1.getRMStateStore();
+
MockNM nm1 =
new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
nm1.registerNode();
@@ -735,7 +739,6 @@ public class TestAMRestart {
RMAppImpl app1 = (RMAppImpl)rm1.submitApp(200, 10000, false);
app1.setSystemClock(clock);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
-
// Fail attempt1 normally
nm1.nodeHeartbeat(am1.getApplicationAttemptId(),
1, ContainerState.COMPLETE);
@@ -771,8 +774,12 @@ public class TestAMRestart {
@SuppressWarnings("resource")
MockRM rm2 = new MockRM(conf, memStore);
rm2.start();
+
+ MockMemoryRMStateStore memStore1 =
+ (MockMemoryRMStateStore) rm2.getRMStateStore();
ApplicationStateData app1State =
- memStore.getState().getApplicationState().get(app1.getApplicationId());
+ memStore1.getState().getApplicationState().
+ get(app1.getApplicationId());
Assert.assertEquals(1, app1State.getFirstAttemptId());
// re-register the NM
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 7c54b60..344f8bb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
@@ -28,17 +29,23 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.Credentials;
@@ -46,14 +53,22 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -251,7 +266,113 @@ public class TestRMAppTransitions {
rmDispatcher.start();
}
- protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) {
+ private ByteBuffer getTokens() throws IOException {
+ Credentials ts = new Credentials();
+ DataOutputBuffer dob = new DataOutputBuffer();
+ ts.writeTokenStorageToStream(dob);
+ ByteBuffer securityTokens =
+ ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ return securityTokens;
+ }
+
+ private ByteBuffer getTokensConf() throws IOException {
+
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Configuration appConf = new Configuration(false);
+ appConf.clear();
+ appConf.set("dfs.nameservices", "mycluster1,mycluster2");
+ appConf.set("dfs.namenode.rpc-address.mycluster2.nn1",
+ "123.0.0.1");
+ appConf.set("dfs.namenode.rpc-address.mycluster3.nn2",
+ "123.0.0.2");
+ appConf.write(dob);
+ ByteBuffer tokenConf =
+ ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ return tokenConf;
+ }
+
+ private Map<String, LocalResource> getLocalResources()
+ throws UnsupportedFileSystemException {
+ FileContext localFS = FileContext.getLocalFSFileContext();
+ File tmpDir = new File("target");
+ File scriptFile = new File(tmpDir, "scriptFile.sh");
+ URL resourceURL =
+ URL.fromPath(localFS
+ .makeQualified(new Path(scriptFile.getAbsolutePath())));
+ LocalResource localRes =
+ Records.newRecord(LocalResource.class);
+ localRes.setResource(resourceURL);
+ localRes.setSize(-1);
+ localRes.setVisibility(LocalResourceVisibility.APPLICATION);
+ localRes.setType(LocalResourceType.FILE);
+ localRes.setTimestamp(scriptFile.lastModified());
+ String destinationFile = "dest_file";
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(destinationFile, localRes);
+ return localResources;
+ }
+
+ private Map<String, String> getEnvironment() {
+ Map<String, String> userSetEnv = new HashMap<String, String>();
+ userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
+ userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST");
+ userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
+ userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
+ userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
+ userSetEnv.put(Environment.USER.key(), "user_set_" +
+ Environment.USER.key());
+ userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
+ userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
+ userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
+ return userSetEnv;
+ }
+
+ private ContainerRetryContext getContainerRetryContext() {
+ ContainerRetryContext containerRetryContext = ContainerRetryContext
+ .newInstance(
+ ContainerRetryPolicy.RETRY_ON_SPECIFIC_ERROR_CODES,
+ new HashSet<>(Arrays.asList(Integer.valueOf(111))), 0, 0);
+ return containerRetryContext;
+ }
+
+ private Map<String, ByteBuffer> getServiceData() {
+ Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
+ String serviceName = "non_exist_auxService";
+ serviceData.put(serviceName, ByteBuffer.wrap(serviceName.getBytes()));
+ return serviceData;
+ }
+
+ private ContainerLaunchContext prepareContainerLaunchContext()
+ throws IOException {
+
+ ContainerLaunchContext clc =
+ Records.newRecord(ContainerLaunchContext.class);
+ clc.setCommands(Arrays.asList("/bin/sleep 5"));
+ if (UserGroupInformation.isSecurityEnabled()) {
+ clc.setTokens(getTokens());
+ clc.setTokensConf(getTokensConf());
+ }
+ clc.setLocalResources(getLocalResources());
+ clc.setEnvironment(getEnvironment());
+ clc.setContainerRetryContext(getContainerRetryContext());
+ clc.setServiceData(getServiceData());
+ return clc;
+ }
+
+ private LogAggregationContext getLogAggregationContext() {
+ LogAggregationContext logAggregationContext =
+ LogAggregationContext.newInstance(
+ "includePattern", "excludePattern",
+ "rolledLogsIncludePattern",
+ "rolledLogsExcludePattern",
+ "policyClass",
+ "policyParameters");
+ return logAggregationContext;
+ }
+
+ protected RMApp createNewTestApp(ApplicationSubmissionContext
+ submissionContext) throws IOException {
ApplicationId applicationId = MockApps.newAppID(appId++);
String user = MockApps.newUserName();
String name = MockApps.newAppName();
@@ -270,7 +391,9 @@ public class TestRMAppTransitions {
// but applicationId is still set for safety
submissionContext.setApplicationId(applicationId);
submissionContext.setPriority(Priority.newInstance(0));
- submissionContext.setAMContainerSpec(mock(ContainerLaunchContext.class));
+ submissionContext.setAMContainerSpec(prepareContainerLaunchContext());
+ submissionContext.setLogAggregationContext(getLogAggregationContext());
+
RMApp application = new RMAppImpl(applicationId, rmContext, conf, name,
user, queue, submissionContext, scheduler, masterService,
System.currentTimeMillis(), "YARN", null,
@@ -405,6 +528,7 @@ public class TestRMAppTransitions {
// verify sendATSCreateEvent() is get called during
// AddApplicationToSchedulerTransition.
verify(publisher).appCreated(eq(application), anyLong());
+ verifyRMAppFieldsForNonFinalTransitions(application);
return application;
}
@@ -422,6 +546,7 @@ public class TestRMAppTransitions {
application.handle(event);
assertStartTimeSet(application);
assertAppState(RMAppState.SUBMITTED, application);
+ verifyRMAppFieldsForNonFinalTransitions(application);
return application;
}
@@ -530,6 +655,7 @@ public class TestRMAppTransitions {
assertFailed(application,
".*Unmanaged application.*Failing the application.*");
assertAppFinalStateSaved(application);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test
@@ -539,6 +665,7 @@ public class TestRMAppTransitions {
RMApp application = testCreateAppFinished(null, diagMsg);
Assert.assertTrue("Finished application missing diagnostics",
application.getDiagnostics().indexOf(diagMsg) != -1);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test (timeout = 30000)
@@ -546,15 +673,7 @@ public class TestRMAppTransitions {
LOG.info("--- START: testAppRecoverPath ---");
ApplicationSubmissionContext sub =
Records.newRecord(ApplicationSubmissionContext.class);
- ContainerLaunchContext clc =
- Records.newRecord(ContainerLaunchContext.class);
- Credentials credentials = new Credentials();
- DataOutputBuffer dob = new DataOutputBuffer();
- credentials.writeTokenStorageToStream(dob);
- ByteBuffer securityTokens =
- ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- clc.setTokens(securityTokens);
- sub.setAMContainerSpec(clc);
+ sub.setAMContainerSpec(prepareContainerLaunchContext());
testCreateAppSubmittedRecovery(sub);
}
@@ -577,6 +696,7 @@ public class TestRMAppTransitions {
assertAppFinalStateNotSaved(application);
verifyApplicationFinished(RMAppState.KILLED);
verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test
@@ -594,6 +714,7 @@ public class TestRMAppTransitions {
assertFailed(application, rejectedText);
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test (timeout = 30000)
@@ -611,6 +732,7 @@ public class TestRMAppTransitions {
assertFailed(application, rejectedText);
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED);
+ verifyRMAppFieldsForFinalTransitions(application);
rmContext.getStateStore().removeApplication(application);
}
@@ -633,6 +755,7 @@ public class TestRMAppTransitions {
assertKilled(application);
verifyApplicationFinished(RMAppState.KILLED);
verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test (timeout = 30000)
@@ -650,6 +773,7 @@ public class TestRMAppTransitions {
assertFailed(application, rejectedText);
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test (timeout = 30000)
@@ -684,6 +808,7 @@ public class TestRMAppTransitions {
assertFailed(application, rejectedText);
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test
@@ -706,6 +831,7 @@ public class TestRMAppTransitions {
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.KILLED);
verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test
@@ -769,8 +895,9 @@ public class TestRMAppTransitions {
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.KILLED);
verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
-
+
@Test
public void testAppAcceptedAttemptKilled() throws IOException,
InterruptedException {
@@ -816,6 +943,7 @@ public class TestRMAppTransitions {
assertKilled(application);
verifyApplicationFinished(RMAppState.KILLED);
verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test
@@ -873,6 +1001,7 @@ public class TestRMAppTransitions {
assertFailed(application, ".*Failing the application.*");
assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test
@@ -914,6 +1043,7 @@ public class TestRMAppTransitions {
assertFinalAppStatus(FinalApplicationStatus.FAILED, application);
Assert.assertTrue("Finished app missing diagnostics", application
.getDiagnostics().indexOf(diagMsg) != -1);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test
@@ -933,6 +1063,7 @@ public class TestRMAppTransitions {
Assert.assertEquals("application diagnostics is not correct",
"", diag.toString());
verifyApplicationFinished(RMAppState.FINISHED);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test (timeout = 30000)
@@ -962,6 +1093,7 @@ public class TestRMAppTransitions {
assertTimesAtFinish(application);
assertAppState(RMAppState.FAILED, application);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test (timeout = 30000)
@@ -1016,6 +1148,7 @@ public class TestRMAppTransitions {
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
+ verifyRMAppFieldsForFinalTransitions(application);
}
@Test(timeout = 30000)
@@ -1061,11 +1194,12 @@ public class TestRMAppTransitions {
RMAppState finalState = appState.getState();
Assert.assertEquals("Application is not in finalState.", finalState,
application.getState());
+ verifyRMAppFieldsForFinalTransitions(application);
}
public void createRMStateForApplications(
Map<ApplicationId, ApplicationStateData> applicationState,
- RMAppState rmAppState) {
+ RMAppState rmAppState) throws IOException {
RMApp app = createNewTestApp(null);
ApplicationStateData appState =
ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(),
@@ -1075,7 +1209,7 @@ public class TestRMAppTransitions {
}
@Test
- public void testGetAppReport() {
+ public void testGetAppReport() throws IOException {
RMApp app = createNewTestApp(null);
assertAppState(RMAppState.NEW, app);
ApplicationReport report = app.createAndGetApplicationReport(null, true);
@@ -1109,4 +1243,41 @@ public class TestRMAppTransitions {
Assert.assertEquals(finalState, appRemovedEvent.getFinalState());
}
}
+
+ private void verifyRMAppFieldsForNonFinalTransitions(RMApp application)
+ throws IOException {
+ assertEquals(Arrays.asList("/bin/sleep 5"),
+ application.getApplicationSubmissionContext().
+ getAMContainerSpec().getCommands());
+ assertEquals(getLocalResources(),
+ application.getApplicationSubmissionContext().
+ getAMContainerSpec().getLocalResources());
+ if(UserGroupInformation.isSecurityEnabled()) {
+ assertEquals(getTokens(),
+ application.getApplicationSubmissionContext().
+ getAMContainerSpec().getTokens());
+ assertEquals(getTokensConf(),
+ application.getApplicationSubmissionContext().
+ getAMContainerSpec().getTokensConf());
+ }
+ assertEquals(getEnvironment(),
+ application.getApplicationSubmissionContext().
+ getAMContainerSpec().getEnvironment());
+ assertEquals(getContainerRetryContext(),
+ application.getApplicationSubmissionContext().
+ getAMContainerSpec().getContainerRetryContext());
+ assertEquals(getServiceData(),
+ application.getApplicationSubmissionContext().
+ getAMContainerSpec().getServiceData());
+ assertEquals(getLogAggregationContext(),
+ application.getApplicationSubmissionContext().
+ getLogAggregationContext());
+ }
+
+ private void verifyRMAppFieldsForFinalTransitions(RMApp application) {
+ assertEquals(null, application.getApplicationSubmissionContext().
+ getAMContainerSpec());
+ assertEquals(null, application.getApplicationSubmissionContext().
+ getLogAggregationContext());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5a7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
index 640293c..2c52377 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.MockRMMemoryStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.MockMemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM;
@@ -78,7 +78,7 @@ public class TestRMDelegationTokens {
UserGroupInformation.getLoginUser()
.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
- MemoryRMStateStore memStore = new MockRMMemoryStateStore();
+ MemoryRMStateStore memStore = new MockMemoryRMStateStore();
memStore.init(conf);
RMState rmState = memStore.getState();
@@ -132,7 +132,7 @@ public class TestRMDelegationTokens {
// Test all expired keys are removed from state-store.
@Test(timeout = 15000)
public void testRemoveExpiredMasterKeyInRMStateStore() throws Exception {
- MemoryRMStateStore memStore = new MockRMMemoryStateStore();
+ MemoryRMStateStore memStore = new MockMemoryRMStateStore();
memStore.init(testConf);
RMState rmState = memStore.getState();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDFS-12271. Incorrect statement in
Downgrade section of HDFS Rolling Upgrade document. Contributed by
Nandakumar.
Posted by kk...@apache.org.
HDFS-12271. Incorrect statement in Downgrade section of HDFS Rolling Upgrade document. Contributed by Nandakumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c62ff7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c62ff7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c62ff7f
Branch: refs/heads/YARN-6592
Commit: 2c62ff7fd4392d55fe41573da297e2c0bc4f4c41
Parents: 442ea85
Author: Chen Liang <cl...@apache.org>
Authored: Mon Oct 2 13:18:00 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Oct 2 13:18:00 2017 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c62ff7f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
index cabdd74..386ab6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
@@ -179,7 +179,7 @@ Below are the steps for rolling downgrade without downtime:
1. Start *NN2* as standby normally.
1. Failover from *NN1* to *NN2*
so that *NN2* becomes active and *NN1* becomes standby.
- 1. Shutdown and upgrade *NN1*.
+ 1. Shutdown and downgrade *NN1*.
1. Start *NN1* as standby normally.
1. Finalize Rolling Downgrade
1. Run "[`hdfs dfsadmin -rollingUpgrade finalize`](#dfsadmin_-rollingUpgrade)"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-12569. Unset EC policy logs
empty payload in edit log. (Lei (Eddy) Xu)
Posted by kk...@apache.org.
HDFS-12569. Unset EC policy logs empty payload in edit log. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27ffd43b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27ffd43b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27ffd43b
Branch: refs/heads/YARN-6592
Commit: 27ffd43b6419c9ebe697536bcb6abb858ce791d2
Parents: 280080f
Author: Lei Xu <le...@apache.org>
Authored: Mon Oct 2 15:31:20 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Mon Oct 2 15:35:49 2017 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirErasureCodingOp.java | 3 +--
.../hadoop/hdfs/server/namenode/FSDirXAttrOp.java | 4 ++++
.../hdfs/TestUnsetAndChangeDirectoryEcPolicy.java | 13 +++++++++++++
3 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffd43b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 181b147..391e392 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -298,8 +298,7 @@ final class FSDirErasureCodingOp {
final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
xattrs.add(ecXAttr);
- FSDirXAttrOp.unprotectedRemoveXAttrs(fsd, srcIIP, xattrs);
- return xattrs;
+ return FSDirXAttrOp.unprotectedRemoveXAttrs(fsd, srcIIP, xattrs);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffd43b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index acdade7..3223467 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -184,6 +184,10 @@ class FSDirXAttrOp {
return fsd.getAuditFileInfo(iip);
}
+ /**
+ * Remove xattrs from the inode, and return the <em>removed</em> xattrs.
+ * @return the <em>removed</em> xattrs.
+ */
static List<XAttr> unprotectedRemoveXAttrs(
FSDirectory fsd, final INodesInPath iip, final List<XAttr> toRemove)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ffd43b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index 529a110..52cf163 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -362,4 +362,17 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
+ ecFilePath, e);
}
}
+
+ /**
+ * Test unsetEcPolicy is persisted correctly in edit log.
+ */
+ @Test
+ public void testUnsetEcPolicyInEditLog() throws IOException {
+ fs.getClient().setErasureCodingPolicy("/", ecPolicy.getName());
+ Assert.assertEquals(ecPolicy, fs.getErasureCodingPolicy(new Path("/")));
+ fs.getClient().unsetErasureCodingPolicy("/");
+
+ cluster.restartNameNode(true);
+ Assert.assertNull(fs.getErasureCodingPolicy(new Path("/")));
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HADOOP-14915. method name is
incorrect in ConfServlet. Contributed by Bharat Viswanadham.
Posted by kk...@apache.org.
HADOOP-14915. method name is incorrect in ConfServlet. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/563dcdfc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/563dcdfc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/563dcdfc
Branch: refs/heads/YARN-6592
Commit: 563dcdfc1de7ea9ee7ce296163cf2678dfe5349c
Parents: 06df6ab
Author: Chen Liang <cl...@apache.org>
Authored: Mon Oct 2 10:58:19 2017 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Oct 2 10:58:19 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/conf/ConfServlet.java | 4 ++--
.../src/test/java/org/apache/hadoop/conf/TestConfServlet.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/563dcdfc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
index cfd7b97..2128de7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
@@ -69,7 +69,7 @@ public class ConfServlet extends HttpServlet {
return;
}
- String format = parseAccecptHeader(request);
+ String format = parseAcceptHeader(request);
if (FORMAT_XML.equals(format)) {
response.setContentType("text/xml; charset=utf-8");
} else if (FORMAT_JSON.equals(format)) {
@@ -89,7 +89,7 @@ public class ConfServlet extends HttpServlet {
}
@VisibleForTesting
- static String parseAccecptHeader(HttpServletRequest request) {
+ static String parseAcceptHeader(HttpServletRequest request) {
String format = request.getHeader(HttpHeaders.ACCEPT);
return format != null && format.contains(FORMAT_JSON) ?
FORMAT_JSON : FORMAT_XML;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/563dcdfc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
index 53089ed..cf42219 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
@@ -98,7 +98,7 @@ public class TestConfServlet {
Mockito.when(request.getHeader(HttpHeaders.ACCEPT))
.thenReturn(contentTypeExpected);
assertEquals(contenTypeActual,
- ConfServlet.parseAccecptHeader(request));
+ ConfServlet.parseAcceptHeader(request));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-7084.
TestSchedulingMonitor#testRMStarts fails sporadically. Contributed by Jason
Lowe.
Posted by kk...@apache.org.
YARN-7084. TestSchedulingMonitor#testRMStarts fails sporadically. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f6118f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f6118f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f6118f9
Branch: refs/heads/YARN-6592
Commit: 7f6118f918e0fbe81cb64e8b9f6959b8d66679f6
Parents: d3b1c63
Author: Eric Payne <ep...@apache.org>
Authored: Fri Sep 29 10:33:57 2017 -0500
Committer: Eric Payne <ep...@apache.org>
Committed: Fri Sep 29 10:33:57 2017 -0500
----------------------------------------------------------------------
.../monitor/TestSchedulingMonitor.java | 34 +++++++-------------
1 file changed, 11 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f6118f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
index d6ac346..c38236d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
@@ -25,42 +25,30 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
import org.junit.Test;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class TestSchedulingMonitor {
@Test(timeout = 10000)
- public void testRMStarts() {
+ public void testRMStarts() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
ResourceManager rm = new MockRM();
- try {
- rm.init(conf);
- } catch (Exception e) {
- fail("ResourceManager does not start when " +
- YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + " is set to true");
- }
-
+ rm.init(conf);
SchedulingEditPolicy mPolicy = mock(SchedulingEditPolicy.class);
when(mPolicy.getMonitoringInterval()).thenReturn(1000L);
SchedulingMonitor monitor = new SchedulingMonitor(rm.getRMContext(),
mPolicy);
- try {
- monitor.serviceInit(conf);
- monitor.serviceStart();
- } catch (Exception e) {
- fail("SchedulingMonitor failes to start.");
- }
- verify(mPolicy, times(1)).editSchedule();
- try {
- monitor.close();
- rm.close();
- } catch (Exception e) {
- fail("Failed to close.");
- }
+ monitor.serviceInit(conf);
+ monitor.serviceStart();
+ verify(mPolicy, timeout(10000)).editSchedule();
+ monitor.close();
+ rm.close();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-7226. Whitelisted variables
do not support delayed variable expansion. Contributed by Jason Lowe
Posted by kk...@apache.org.
YARN-7226. Whitelisted variables do not support delayed variable expansion. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eb84999
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eb84999
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eb84999
Branch: refs/heads/YARN-6592
Commit: 7eb8499996869cdb63743f1c9eca2ba91d57ad08
Parents: d4d2fd1
Author: Sidharta S <si...@apache.org>
Authored: Mon Oct 2 19:04:49 2017 -0700
Committer: Sidharta S <si...@apache.org>
Committed: Mon Oct 2 19:04:49 2017 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 44 ++++++++----
.../nodemanager/LinuxContainerExecutor.java | 8 +++
.../launcher/ContainerLaunch.java | 25 -------
.../runtime/DefaultLinuxContainerRuntime.java | 6 ++
.../DelegatingLinuxContainerRuntime.java | 11 +++
.../runtime/DockerLinuxContainerRuntime.java | 7 ++
.../runtime/ContainerRuntime.java | 11 +++
.../launcher/TestContainerLaunch.java | 73 ++++++++++++++++++--
8 files changed, 141 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 5fd059d..0b4dbc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -26,10 +26,8 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -95,10 +93,15 @@ public abstract class ContainerExecutor implements Configurable {
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private final ReadLock readLock = lock.readLock();
private final WriteLock writeLock = lock.writeLock();
+ private String[] whitelistVars;
@Override
public void setConf(Configuration conf) {
this.conf = conf;
+ if (conf != null) {
+ whitelistVars = conf.get(YarnConfiguration.NM_ENV_WHITELIST,
+ YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");
+ }
}
@Override
@@ -331,6 +334,8 @@ public abstract class ContainerExecutor implements Configurable {
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
Map<Path, List<String>> resources, List<String> command, Path logDir,
String user, String outFilename) throws IOException {
+ updateEnvForWhitelistVars(environment);
+
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
@@ -341,22 +346,11 @@ public abstract class ContainerExecutor implements Configurable {
sb.stdout(logDir, CONTAINER_PRE_LAUNCH_STDOUT);
sb.stderr(logDir, CONTAINER_PRE_LAUNCH_STDERR);
- Set<String> whitelist = new HashSet<>();
-
- String[] nmWhiteList = conf.get(YarnConfiguration.NM_ENV_WHITELIST,
- YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");
- for (String param : nmWhiteList) {
- whitelist.add(param);
- }
if (environment != null) {
sb.echo("Setting up env variables");
for (Map.Entry<String, String> env : environment.entrySet()) {
- if (!whitelist.contains(env.getKey())) {
- sb.env(env.getKey(), env.getValue());
- } else {
- sb.whitelistedEnv(env.getKey(), env.getValue());
- }
+ sb.env(env.getKey(), env.getValue());
}
}
@@ -658,6 +652,28 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
+ * Propagate variables from the nodemanager's environment into the
+ * container's environment if unspecified by the container.
+ * @param env the environment to update
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ for(String var : whitelistVars) {
+ if (!env.containsKey(var)) {
+ String val = getNMEnvVar(var);
+ if (val != null) {
+ env.put(var, val);
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ protected String getNMEnvVar(String varname) {
+ return System.getenv(varname);
+ }
+
+ /**
* Mark the container as active.
*
* @param containerId the container ID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 9e004e7..9f1ec60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -62,6 +62,7 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -442,6 +443,13 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ if (linuxContainerRuntime.useWhitelistEnv(env)) {
+ super.updateEnvForWhitelistVars(env);
+ }
+ }
+
+ @Override
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
Container container = ctx.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f929dfc..f63b1b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -89,7 +89,6 @@ import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -1022,8 +1021,6 @@ public class ContainerLaunch implements Callable<Integer> {
public abstract void command(List<String> command) throws IOException;
- public abstract void whitelistedEnv(String key, String value) throws IOException;
-
protected static final String ENV_PRELAUNCH_STDOUT = "PRELAUNCH_OUT";
protected static final String ENV_PRELAUNCH_STDERR = "PRELAUNCH_ERR";
@@ -1163,11 +1160,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
- public void whitelistedEnv(String key, String value) throws IOException {
- line("export ", key, "=${", key, ":-", "\"", value, "\"}");
- }
-
- @Override
public void setStdOut(final Path stdout) throws IOException {
line("export ", ENV_PRELAUNCH_STDOUT, "=\"", stdout.toString(), "\"");
// tee is needed for DefaultContainerExecutor error propagation to stdout
@@ -1262,12 +1254,6 @@ public class ContainerLaunch implements Callable<Integer> {
errorCheck();
}
- @Override
- public void whitelistedEnv(String key, String value) throws IOException {
- lineWithLenCheck("@set ", key, "=", value);
- errorCheck();
- }
-
//Dummy implementation
@Override
protected void setStdOut(final Path stdout) throws IOException {
@@ -1389,17 +1375,6 @@ public class ContainerLaunch implements Callable<Integer> {
environment.put("JVM_PID", "$$");
}
- /**
- * Modifiable environment variables
- */
-
- // allow containers to override these variables
- String[] whitelist = conf.get(YarnConfiguration.NM_ENV_WHITELIST, YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");
-
- for(String whitelistEnvVariable : whitelist) {
- putEnvIfAbsent(environment, whitelistEnvVariable.trim());
- }
-
// variables here will be forced in, even if the container has specified them.
Apps.setEnvFromInputString(environment, conf.get(
YarnConfiguration.NM_ADMIN_USER_ENV,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 6c3ae85..e9c58b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -36,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -72,6 +73,11 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ return true;
+ }
+
+ @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
//nothing to do here at the moment.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 9fe4927..517a4e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -93,6 +93,17 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ @Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ try {
+ LinuxContainerRuntime runtime = pickContainerRuntime(env);
+ return runtime.useWhitelistEnv(env);
+ } catch (ContainerExecutionException e) {
+ LOG.debug("Unable to determine runtime");
+ return false;
+ }
+ }
+
@VisibleForTesting
LinuxContainerRuntime pickContainerRuntime(
Map<String, String> environment) throws ContainerExecutionException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 29c6fe9..2013306 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -282,6 +282,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ // Avoid propagating nodemanager environment variables into the container
+ // so those variables can be picked up from the Docker image instead.
+ return false;
+ }
+
+ @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index 7caa0ed..aa294fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import java.util.Map;
+
/**
* An abstraction for various container runtime implementations. Examples
* include Process Tree, Docker, Appc runtimes etc. These implementations
@@ -83,4 +85,13 @@ public interface ContainerRuntime {
* and hostname
*/
String[] getIpAndHost(Container container) throws ContainerExecutionException;
+
+ /**
+ * Whether to propagate the whitelist of environment variables from the
+ * nodemanager environment into the container environment.
+ * @param env the container's environment variables
+ * @return true if whitelist variables should be propagated, false otherwise
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ boolean useWhitelistEnv(Map<String, String> env);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eb84999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 588fdcd..1ee6eb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -99,6 +100,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.ShellScriptBuilder;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
@@ -304,8 +307,18 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
FileOutputStream fos = new FileOutputStream(shellFile);
List<String> commands = new ArrayList<String>();
+ final Map<String, String> nmEnv = new HashMap<>();
+ nmEnv.put("HADOOP_COMMON_HOME", "nodemanager_common_home");
+ nmEnv.put("HADOOP_HDFS_HOME", "nodemanager_hdfs_home");
+ nmEnv.put("HADOOP_YARN_HOME", "nodemanager_yarn_home");
+ nmEnv.put("HADOOP_MAPRED_HOME", "nodemanager_mapred_home");
DefaultContainerExecutor defaultContainerExecutor =
- new DefaultContainerExecutor();
+ new DefaultContainerExecutor() {
+ @Override
+ protected String getNMEnvVar(String varname) {
+ return nmEnv.get(varname);
+ }
+ };
YarnConfiguration conf = new YarnConfiguration();
conf.set(YarnConfiguration.NM_ENV_WHITELIST,
"HADOOP_MAPRED_HOME,HADOOP_YARN_HOME");
@@ -317,10 +330,60 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
StandardCharsets.UTF_8);
Assert.assertTrue(shellContent
.contains("export HADOOP_COMMON_HOME=\"/opt/hadoopcommon\""));
- // Not available in env and whitelist
- Assert.assertTrue(shellContent.contains("export HADOOP_MAPRED_HOME="
- + "${HADOOP_MAPRED_HOME:-\"/opt/hadoopbuild\"}"));
- // Not available in env but in whitelist
+ // Whitelisted variable overridden by container
+ Assert.assertTrue(shellContent.contains(
+ "export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\""));
+ // Available in env but not in whitelist
+ Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
+ // Available in env and in whitelist
+ Assert.assertTrue(shellContent.contains(
+ "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\""));
+ fos.flush();
+ fos.close();
+ }
+
+ @Test(timeout = 20000)
+ public void testWriteEnvExportDocker() throws Exception {
+ // Valid only for unix
+ assumeNotWindows();
+ File shellFile = Shell.appendScriptExtension(tmpDir, "hello");
+ Map<String, String> env = new HashMap<String, String>();
+ env.put("HADOOP_COMMON_HOME", "/opt/hadoopcommon");
+ env.put("HADOOP_MAPRED_HOME", "/opt/hadoopbuild");
+ Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
+ FileOutputStream fos = new FileOutputStream(shellFile);
+ List<String> commands = new ArrayList<String>();
+ final Map<String, String> nmEnv = new HashMap<>();
+ nmEnv.put("HADOOP_COMMON_HOME", "nodemanager_common_home");
+ nmEnv.put("HADOOP_HDFS_HOME", "nodemanager_hdfs_home");
+ nmEnv.put("HADOOP_YARN_HOME", "nodemanager_yarn_home");
+ nmEnv.put("HADOOP_MAPRED_HOME", "nodemanager_mapred_home");
+ DockerLinuxContainerRuntime dockerRuntime =
+ new DockerLinuxContainerRuntime(
+ mock(PrivilegedOperationExecutor.class));
+ LinuxContainerExecutor lce =
+ new LinuxContainerExecutor(dockerRuntime) {
+ @Override
+ protected String getNMEnvVar(String varname) {
+ return nmEnv.get(varname);
+ }
+ };
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_ENV_WHITELIST,
+ "HADOOP_MAPRED_HOME,HADOOP_YARN_HOME");
+ lce.setConf(conf);
+ lce.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), "user");
+ String shellContent =
+ new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())),
+ StandardCharsets.UTF_8);
+ Assert.assertTrue(shellContent
+ .contains("export HADOOP_COMMON_HOME=\"/opt/hadoopcommon\""));
+ // Whitelisted variable overridden by container
+ Assert.assertTrue(shellContent.contains(
+ "export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\""));
+ // Verify no whitelisted variables inherited from NM env
+ Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME"));
fos.flush();
fos.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDFS-12455. WebHDFS - Adding
"snapshot enabled" status to ListStatus query result. Contributed by Ajay
Kumar.
Posted by kk...@apache.org.
HDFS-12455. WebHDFS - Adding "snapshot enabled" status to ListStatus query result. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/107c1777
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/107c1777
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/107c1777
Branch: refs/heads/YARN-6592
Commit: 107c177782a24a16c66113841f2fc5144f56207b
Parents: b913051
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Oct 3 13:02:12 2017 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Oct 3 13:02:12 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileStatus.java | 27 +++++++++++++++++++-
.../apache/hadoop/fs/protocolPB/PBHelper.java | 14 +++++++---
.../hadoop-common/src/main/proto/FSProtos.proto | 7 ++---
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 12 ++++++++-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 5 ++++
.../src/main/proto/hdfs.proto | 1 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 17 +++++++++---
...tNamenodeProtocolServerSideTranslatorPB.java | 1 +
.../server/namenode/FSDirStatAndListingOp.java | 7 +++++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 4 +++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 2 ++
.../hadoop/hdfs/TestDistributedFileSystem.java | 19 ++++++++++++++
12 files changed, 103 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index 8575439..d7c05d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -60,7 +60,8 @@ public class FileStatus implements Writable, Comparable<Object>,
HAS_ACL,
HAS_CRYPT,
HAS_EC,
- };
+ SNAPSHOT_ENABLED
+ }
private static final Set<AttrFlags> NONE = Collections.<AttrFlags>emptySet();
private static Set<AttrFlags> flags(boolean acl, boolean crypt, boolean ec) {
if (!(acl || crypt || ec)) {
@@ -274,6 +275,15 @@ public class FileStatus implements Writable, Comparable<Object>,
}
/**
+ * Check if directory is Snapshot enabled or not.
+ *
+ * @return true if directory is snapshot enabled
+ */
+ public boolean isSnapshotEnabled() {
+ return attr.contains(AttrFlags.SNAPSHOT_ENABLED);
+ }
+
+ /**
* Get the owner of the file.
* @return owner of the file. The string could be empty if there is no
* notion of owner of a file in a filesystem or if it could not
@@ -331,6 +341,19 @@ public class FileStatus implements Writable, Comparable<Object>,
}
/**
+ * Sets Snapshot enabled flag.
+ *
+ * @param isSnapShotEnabled When true, SNAPSHOT_ENABLED flag is set
+ */
+ public void setSnapShotEnabledFlag(boolean isSnapShotEnabled) {
+ if (isSnapShotEnabled) {
+ attr.add(AttrFlags.SNAPSHOT_ENABLED);
+ } else {
+ attr.remove(AttrFlags.SNAPSHOT_ENABLED);
+ }
+ }
+
+ /**
* @return The contents of the symbolic link.
*/
public Path getSymlink() throws IOException {
@@ -486,4 +509,6 @@ public class FileStatus implements Writable, Comparable<Object>,
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
index aa7d8f5..23caf2e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
@@ -94,11 +94,15 @@ public final class PBHelper {
owner = proto.getOwner();
group = proto.getGroup();
int flags = proto.getFlags();
- return new FileStatus(length, isdir, blockReplication, blocksize,
- mtime, atime, permission, owner, group, symlink, path,
- (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0,
+ FileStatus fileStatus = new FileStatus(length, isdir, blockReplication,
+ blocksize, mtime, atime, permission, owner, group, symlink, path,
+ (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0,
(flags & FileStatusProto.Flags.HAS_CRYPT_VALUE) != 0,
- (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0);
+ (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0);
+
+ fileStatus.setSnapShotEnabledFlag((flags & FileStatusProto.Flags
+ .SNAPSHOT_ENABLED_VALUE) != 0);
+ return fileStatus;
}
public static FileStatusProto convert(FileStatus stat) throws IOException {
@@ -124,6 +128,8 @@ public final class PBHelper {
flags |= stat.hasAcl() ? FileStatusProto.Flags.HAS_ACL_VALUE : 0;
flags |= stat.isEncrypted() ? FileStatusProto.Flags.HAS_CRYPT_VALUE : 0;
flags |= stat.isErasureCoded() ? FileStatusProto.Flags.HAS_EC_VALUE : 0;
+ flags |= stat.isSnapshotEnabled() ? FileStatusProto.Flags
+ .SNAPSHOT_ENABLED_VALUE : 0;
bld.setFlags(flags);
return bld.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
index 6fe7980..5b8c45d 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
@@ -44,9 +44,10 @@ message FileStatusProto {
FT_SYMLINK = 3;
}
enum Flags {
- HAS_ACL = 0x01; // has ACLs
- HAS_CRYPT = 0x02; // encrypted
- HAS_EC = 0x04; // erasure coded
+ HAS_ACL = 0x01; // has ACLs
+ HAS_CRYPT = 0x02; // encrypted
+ HAS_EC = 0x04; // erasure coded
+ SNAPSHOT_ENABLED = 0x08; // snapshot enabled
}
required FileType fileType = 1;
required string path = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index 8438b01..163881f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -56,7 +56,8 @@ public class HdfsFileStatus extends FileStatus {
public enum Flags {
HAS_ACL,
HAS_CRYPT,
- HAS_EC;
+ HAS_EC,
+ SNAPSHOT_ENABLED
}
private final EnumSet<Flags> flags;
@@ -244,6 +245,15 @@ public class HdfsFileStatus extends FileStatus {
return storagePolicy;
}
+ /**
+ * Check if directory is Snapshot enabled or not.
+ *
+ * @return true if directory is snapshot enabled
+ */
+ public boolean isSnapshotEnabled() {
+ return flags.contains(Flags.SNAPSHOT_ENABLED);
+ }
+
@Override
public boolean equals(Object o) {
// satisfy findbugs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index db97031..21b1e1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -1602,6 +1602,9 @@ public class PBHelperClient {
case HAS_EC:
f.add(HdfsFileStatus.Flags.HAS_EC);
break;
+ case SNAPSHOT_ENABLED:
+ f.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
+ break;
default:
// ignore unknown
break;
@@ -2155,6 +2158,8 @@ public class PBHelperClient {
int flags = fs.hasAcl() ? HdfsFileStatusProto.Flags.HAS_ACL_VALUE : 0;
flags |= fs.isEncrypted() ? HdfsFileStatusProto.Flags.HAS_CRYPT_VALUE : 0;
flags |= fs.isErasureCoded() ? HdfsFileStatusProto.Flags.HAS_EC_VALUE : 0;
+ flags |= fs.isSnapshotEnabled() ? HdfsFileStatusProto.Flags
+ .SNAPSHOT_ENABLED_VALUE : 0;
builder.setFlags(flags);
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 7769f68..c38a9be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -410,6 +410,7 @@ message HdfsFileStatusProto {
HAS_ACL = 0x01; // has ACLs
HAS_CRYPT = 0x02; // encrypted
HAS_EC = 0x04; // erasure coded
+ SNAPSHOT_ENABLED = 0x08; // SNAPSHOT ENABLED
}
required FileType fileType = 1;
required bytes path = 2; // local name of inode encoded java UTF8
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index b5880e9..3ff7a61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -198,6 +198,7 @@ public class HttpFSFileSystem extends FileSystem
public static final String ENC_BIT_JSON = "encBit";
public static final String EC_BIT_JSON = "ecBit";
+ public static final String SNAPSHOT_BIT_JSON = "seBit";
public static final String DIRECTORY_LISTING_JSON = "DirectoryListing";
public static final String PARTIAL_LISTING_JSON = "partialListing";
@@ -1066,19 +1067,27 @@ public class HttpFSFileSystem extends FileSystem
final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON);
final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON);
final Boolean erasureBit = (Boolean) json.get(EC_BIT_JSON);
+ final Boolean snapshotEnabledBit = (Boolean) json.get(SNAPSHOT_BIT_JSON);
final boolean aBit = (aclBit != null) ? aclBit : false;
final boolean eBit = (encBit != null) ? encBit : false;
final boolean ecBit = (erasureBit != null) ? erasureBit : false;
- if (aBit || eBit || ecBit) {
+ final boolean seBit =
+ (snapshotEnabledBit != null) ? snapshotEnabledBit : false;
+ if (aBit || eBit || ecBit || seBit) {
// include this for compatibility with 2.x
FsPermissionExtension deprecatedPerm =
new FsPermissionExtension(permission, aBit, eBit, ecBit);
- return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
+ FileStatus fileStatus = new FileStatus(len, FILE_TYPE.DIRECTORY == type,
replication, blockSize, mTime, aTime, deprecatedPerm, owner, group,
null, path, aBit, eBit, ecBit);
+ if (seBit) {
+ fileStatus.setSnapShotEnabledFlag(seBit);
+ }
+ return fileStatus;
+ } else {
+ return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
+ replication, blockSize, mTime, aTime, permission, owner, group, path);
}
- return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
- replication, blockSize, mTime, aTime, permission, owner, group, path);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 0527b3b..af0b154 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -948,6 +948,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+
public GetFileLinkInfoResponseProto getFileLinkInfo(RpcController controller,
GetFileLinkInfoRequestProto req) throws ServiceException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 3b3368d..7c0ba2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -413,6 +413,8 @@ class FSDirStatAndListingOp {
.unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip);
final boolean isErasureCoded = (ecPolicy != null);
+ boolean isSnapShottable = false;
+
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
@@ -433,6 +435,8 @@ class FSDirStatAndListingOp {
loc = new LocatedBlocks();
}
}
+ } else if (node.isDirectory()) {
+ isSnapShottable = node.asDirectory().isSnapshottable();
}
int childrenNum = node.isDirectory() ?
@@ -451,6 +455,9 @@ class FSDirStatAndListingOp {
if (isErasureCoded) {
flags.add(HdfsFileStatus.Flags.HAS_EC);
}
+ if(isSnapShottable){
+ flags.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
+ }
return createFileStatus(
size,
node.isDirectory(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 7fa818a..eae6adb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -135,6 +135,10 @@ public class JsonUtil {
if (status.isErasureCoded()) {
m.put("ecBit", true);
}
+ if (status.isSnapshotEnabled()) {
+ m.put("snapshotEnabled", status.isSnapshotEnabled());
+ }
+
m.put("accessTime", status.getAccessTime());
m.put("modificationTime", status.getModificationTime());
m.put("blockSize", status.getBlockSize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 84e8a57..73c6558 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -440,6 +440,7 @@ See also: [`newlength`](#New_Length), [FileSystem](../../api/org/apache/hadoop/f
"pathSuffix" : "",
"permission" : "777",
"replication" : 0,
+ "snapshotEnabled" : true
"type" : "DIRECTORY" //enum {FILE, DIRECTORY, SYMLINK}
}
}
@@ -485,6 +486,7 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
"pathSuffix" : "bar",
"permission" : "711",
"replication" : 0,
+ "snapshotEnabled" : true
"type" : "DIRECTORY"
},
...
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 987992e..6389f0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1269,6 +1269,25 @@ public class TestDistributedFileSystem {
}
}
+ @Test
+ public void testListStatusOfSnapshotDirs() throws IOException {
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration())
+ .build();
+ try {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ dfs.create(new Path("/parent/test1/dfsclose/file-0"));
+ Path snapShotDir = new Path("/parent/test1/");
+ dfs.allowSnapshot(snapShotDir);
+
+ FileStatus status = dfs.getFileStatus(new Path("/parent/test1"));
+ assertTrue(status.isSnapshotEnabled());
+ status = dfs.getFileStatus(new Path("/parent/"));
+ assertFalse(status.isSnapshotEnabled());
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
@Test(timeout=10000)
public void testDFSClientPeerReadTimeout() throws IOException {
final int timeout = 1000;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-6623. Add support to turn off
launching privileged containers in the container-executor. (Varun Vasudev via
wangda)
Posted by kk...@apache.org.
YARN-6623. Add support to turn off launching privileged containers in the container-executor. (Varun Vasudev via wangda)
Change-Id: I76aec68cbfb42d239279d7245a03290d691e87a4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3b1c631
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3b1c631
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3b1c631
Branch: refs/heads/YARN-6592
Commit: d3b1c6319546706c41a2011ead6c3fe208883200
Parents: c114da5
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Sep 28 16:41:09 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Sep 28 16:41:09 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn/conf/container-executor.cfg | 12 +
.../src/CMakeLists.txt | 5 +-
.../runtime/DockerLinuxContainerRuntime.java | 11 +-
.../linux/runtime/docker/DockerClient.java | 25 +-
.../linux/runtime/docker/DockerCommand.java | 55 +-
.../runtime/docker/DockerCommandExecutor.java | 3 +-
.../runtime/docker/DockerInspectCommand.java | 13 +-
.../linux/runtime/docker/DockerLoadCommand.java | 2 +-
.../linux/runtime/docker/DockerPullCommand.java | 2 +-
.../linux/runtime/docker/DockerRmCommand.java | 4 +-
.../linux/runtime/docker/DockerRunCommand.java | 68 +-
.../linux/runtime/docker/DockerStopCommand.java | 6 +-
.../container-executor/impl/configuration.c | 17 +
.../container-executor/impl/configuration.h | 19 +-
.../impl/container-executor.c | 316 +----
.../impl/container-executor.h | 9 -
.../container-executor/impl/get_executable.c | 3 -
.../container-executor/impl/get_executable.h | 29 +
.../main/native/container-executor/impl/main.c | 31 +-
.../impl/modules/common/module-configs.c | 3 +-
.../impl/modules/common/module-configs.h | 1 +
.../main/native/container-executor/impl/util.c | 60 +-
.../main/native/container-executor/impl/util.h | 46 +-
.../container-executor/impl/utils/docker-util.c | 998 ++++++++++++++++
.../container-executor/impl/utils/docker-util.h | 147 +++
.../impl/utils/string-utils.c | 1 -
.../docker-container-executor.cfg | 13 +
.../test/test-container-executor.c | 149 +--
.../native/container-executor/test/test_util.cc | 37 +-
.../test/utils/test-string-utils.cc | 37 +-
.../test/utils/test_docker_util.cc | 1122 ++++++++++++++++++
.../runtime/TestDockerContainerRuntime.java | 398 ++++---
.../docker/TestDockerCommandExecutor.java | 35 +-
.../docker/TestDockerInspectCommand.java | 29 +-
.../runtime/docker/TestDockerLoadCommand.java | 9 +-
.../runtime/docker/TestDockerPullCommand.java | 8 +-
.../runtime/docker/TestDockerRmCommand.java | 8 +-
.../runtime/docker/TestDockerRunCommand.java | 25 +-
.../runtime/docker/TestDockerStopCommand.java | 15 +-
.../src/site/markdown/DockerContainers.md | 33 +-
40 files changed, 3067 insertions(+), 737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d68cee8..023654b 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -2,3 +2,15 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema
banned.users=#comma separated list of users who can not run applications
min.user.id=1000#Prevent other super-users
allowed.system.users=##comma separated list of system users who CAN run applications
+feature.tc.enabled=0
+
+# The configs below deal with settings for Docker
+#[docker]
+# module.enabled=## enable/disable the module. set to "true" to enable, disabled by default
+# docker.binary=/usr/bin/docker
+# docker.allowed.capabilities=## comma seperated capabilities that can be granted, e.g CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
+# docker.allowed.devices=## comma seperated list of devices that can be mounted into a container
+# docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none
+# docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
+# docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
+# docker.privileged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index 3d5b506..9d83bf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -132,6 +132,7 @@ add_library(container
main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
main/native/container-executor/impl/modules/common/module-configs.c
main/native/container-executor/impl/modules/gpu/gpu-module.c
+ main/native/container-executor/impl/utils/docker-util.c
)
add_executable(container-executor
@@ -156,6 +157,7 @@ output_directory(test-container-executor target/usr/local/bin)
# unit tests for container executor
add_executable(cetest
+ main/native/container-executor/impl/get_executable.c
main/native/container-executor/impl/util.c
main/native/container-executor/test/test_configuration.cc
main/native/container-executor/test/test_main.cc
@@ -163,6 +165,7 @@ add_executable(cetest
main/native/container-executor/test/utils/test-path-utils.cc
main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
main/native/container-executor/test/modules/gpu/test-gpu-module.cc
- main/native/container-executor/test/test_util.cc)
+ main/native/container-executor/test/test_util.cc
+ main/native/container-executor/test/utils/test_docker_util.cc)
target_link_libraries(cetest gtest container)
output_directory(cetest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index b8d9b0a..29c6fe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -559,8 +559,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
runCommand.setCapabilities(capabilities);
if(cgroupsRootDirectory != null) {
- runCommand.addMountLocation(cgroupsRootDirectory,
- cgroupsRootDirectory + ":ro", false);
+ runCommand.addReadOnlyMountLocation(cgroupsRootDirectory,
+ cgroupsRootDirectory, false);
}
List<String> allDirs = new ArrayList<>(containerLocalDirs);
@@ -584,7 +584,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
String src = validateMount(dir[0], localizedResources);
String dst = dir[1];
- runCommand.addMountLocation(src, dst + ":ro", true);
+ runCommand.addReadOnlyMountLocation(src, dst, true);
}
}
}
@@ -626,7 +626,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
- LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
+ LOG.info("Docker command used: " + runCommand);
throw new ContainerExecutionException("Launch container failed", e
.getExitCode(), e.getOutput(), e.getErrorOutput());
@@ -757,8 +757,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp.appendArgs(tcCommandFile);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Launching container with cmd: " + runCommand
- .getCommandWithArguments());
+ LOG.debug("Launching container with cmd: " + runCommand);
}
return launchOp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
index 536a22d..77c53a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -23,7 +23,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +34,8 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.util.List;
+import java.util.Map;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -68,10 +70,25 @@ public final class DockerClient {
TMP_FILE_SUFFIX, new
File(tmpDirPath));
- Writer writer = new OutputStreamWriter(new FileOutputStream(dockerCommandFile),
- "UTF-8");
+ Writer writer = new OutputStreamWriter(
+ new FileOutputStream(dockerCommandFile), "UTF-8");
PrintWriter printWriter = new PrintWriter(writer);
- printWriter.print(cmd.getCommandWithArguments());
+ printWriter.println("[docker-command-execution]");
+ for (Map.Entry<String, List<String>> entry :
+ cmd.getDockerCommandWithArguments().entrySet()) {
+ if (entry.getKey().contains("=")) {
+ throw new ContainerExecutionException(
+ "'=' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ if (entry.getValue().contains("\n")) {
+ throw new ContainerExecutionException(
+ "'\\n' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ printWriter.println(" " + entry.getKey() + "=" + StringUtils
+ .join(",", entry.getValue()));
+ }
printWriter.close();
return dockerCommandFile.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
index 3b76a5c..7802209 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.StringUtils;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -35,32 +37,55 @@ import java.util.List;
* e.g 'run', 'load', 'inspect' etc.,
*/
-public abstract class DockerCommand {
+public abstract class DockerCommand {
private final String command;
- private final List<String> commandWithArguments;
+ private final Map<String, List<String>> commandArguments;
protected DockerCommand(String command) {
+ String dockerCommandKey = "docker-command";
this.command = command;
- this.commandWithArguments = new ArrayList<>();
- commandWithArguments.add(command);
+ this.commandArguments = new TreeMap<>();
+ commandArguments.put(dockerCommandKey, new ArrayList<>());
+ commandArguments.get(dockerCommandKey).add(command);
}
- /** Returns the docker sub-command string being used
- * e.g 'run'
+ /**
+ * Returns the docker sub-command string being used
+ * e.g 'run'.
*/
public final String getCommandOption() {
return this.command;
}
- /** Add command commandWithArguments - this method is only meant for use by
- * sub-classes
- * @param arguments to be added
+ /**
+ * Add command commandWithArguments - this method is only meant for use by
+ * sub-classes.
+ *
+ * @param key name of the key to be added
+ * @param value value of the key
*/
- protected final void addCommandArguments(String... arguments) {
- this.commandWithArguments.addAll(Arrays.asList(arguments));
+ protected final void addCommandArguments(String key, String value) {
+ List<String> list = commandArguments.get(key);
+ if (list != null) {
+ list.add(value);
+ return;
+ }
+ list = new ArrayList<>();
+ list.add(value);
+ this.commandArguments.put(key, list);
}
- public String getCommandWithArguments() {
- return StringUtils.join(" ", commandWithArguments);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return Collections.unmodifiableMap(commandArguments);
}
-}
\ No newline at end of file
+
+ @Override
+ public String toString() {
+ StringBuffer ret = new StringBuffer(this.command);
+ for (Map.Entry<String, List<String>> entry : commandArguments.entrySet()) {
+ ret.append(" ").append(entry.getKey());
+ ret.append("=").append(StringUtils.join(",", entry.getValue()));
+ }
+ return ret.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
index 5739912..76b53af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
@@ -88,8 +88,7 @@ public final class DockerCommandExecutor {
dockerOp.disableFailureLogging();
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Running docker command: "
- + dockerCommand.getCommandWithArguments());
+ LOG.debug("Running docker command: " + dockerCommand);
}
try {
String result = privilegedOperationExecutor
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
index 812a35f..d27f74d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
@@ -26,16 +26,14 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
*/
public class DockerInspectCommand extends DockerCommand {
private static final String INSPECT_COMMAND = "inspect";
- private String containerName;
public DockerInspectCommand(String containerName) {
super(INSPECT_COMMAND);
- this.containerName = containerName;
+ super.addCommandArguments("name", containerName);
}
public DockerInspectCommand getContainerStatus() {
- super.addCommandArguments("--format='{{.State.Status}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{.State.Status}}");
return this;
}
@@ -43,9 +41,8 @@ public class DockerInspectCommand extends DockerCommand {
// Be sure to not use space in the argument, otherwise the
// extract_values_delim method in container-executor binary
// cannot parse the arguments correctly.
- super.addCommandArguments("--format='{{range(.NetworkSettings.Networks)}}"
- + "{{.IPAddress}},{{end}}{{.Config.Hostname}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}");
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
index e4d92e0..fa2988c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
@@ -25,6 +25,6 @@ public class DockerLoadCommand extends DockerCommand {
public DockerLoadCommand(String localImageFile) {
super(LOAD_COMMAND);
- super.addCommandArguments("--i=" + localImageFile);
+ super.addCommandArguments("image", localImageFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
index 351e09e..5e6108e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
@@ -25,7 +25,7 @@ public class DockerPullCommand extends DockerCommand {
public DockerPullCommand(String imageName) {
super(PULL_COMMAND);
- super.addCommandArguments(imageName);
+ super.addCommandArguments("image", imageName);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
index b1aea61..dcfe777 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
@@ -25,6 +25,6 @@ public class DockerRmCommand extends DockerCommand {
public DockerRmCommand(String containerName) {
super(RM_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 1e1e6e8..c7bf827 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -20,42 +20,39 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
-import org.apache.hadoop.util.StringUtils;
-
import java.io.File;
-import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class DockerRunCommand extends DockerCommand {
private static final String RUN_COMMAND = "run";
- private final String image;
- private List<String> overrrideCommandWithArgs;
/** The following are mandatory: */
public DockerRunCommand(String containerId, String user, String image) {
super(RUN_COMMAND);
- super.addCommandArguments("--name=" + containerId, "--user=" + user);
- this.image = image;
+ super.addCommandArguments("name", containerId);
+ super.addCommandArguments("user", user);
+ super.addCommandArguments("image", image);
}
public DockerRunCommand removeContainerOnExit() {
- super.addCommandArguments("--rm");
+ super.addCommandArguments("rm", "true");
return this;
}
public DockerRunCommand detachOnRun() {
- super.addCommandArguments("-d");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand setContainerWorkDir(String workdir) {
- super.addCommandArguments("--workdir=" + workdir);
+ super.addCommandArguments("workdir", workdir);
return this;
}
public DockerRunCommand setNetworkType(String type) {
- super.addCommandArguments("--net=" + type);
+ super.addCommandArguments("net", type);
return this;
}
@@ -65,79 +62,80 @@ public class DockerRunCommand extends DockerCommand {
if (!sourceExists && !createSource) {
return this;
}
- super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+ super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
+ return this;
+ }
+
+ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
+ destinationPath, boolean createSource) {
+ boolean sourceExists = new File(sourcePath).exists();
+ if (!sourceExists && !createSource) {
+ return this;
+ }
+ super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
return this;
}
public DockerRunCommand setCGroupParent(String parentPath) {
- super.addCommandArguments("--cgroup-parent=" + parentPath);
+ super.addCommandArguments("cgroup-parent", parentPath);
return this;
}
/* Run a privileged container. Use with extreme care */
public DockerRunCommand setPrivileged() {
- super.addCommandArguments("--privileged");
+ super.addCommandArguments("privileged", "true");
return this;
}
public DockerRunCommand setCapabilities(Set<String> capabilties) {
//first, drop all capabilities
- super.addCommandArguments("--cap-drop=ALL");
+ super.addCommandArguments("cap-drop", "ALL");
//now, add the capabilities supplied
for (String capability : capabilties) {
- super.addCommandArguments("--cap-add=" + capability);
+ super.addCommandArguments("cap-add", capability);
}
return this;
}
public DockerRunCommand setHostname(String hostname) {
- super.addCommandArguments("--hostname=" + hostname);
+ super.addCommandArguments("hostname", hostname);
return this;
}
public DockerRunCommand addDevice(String sourceDevice, String
destinationDevice) {
- super.addCommandArguments("--device=" + sourceDevice + ":" +
+ super.addCommandArguments("devices", sourceDevice + ":" +
destinationDevice);
return this;
}
public DockerRunCommand enableDetach() {
- super.addCommandArguments("--detach=true");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand disableDetach() {
- super.addCommandArguments("--detach=false");
+ super.addCommandArguments("detach", "false");
return this;
}
public DockerRunCommand groupAdd(String[] groups) {
- for(int i = 0; i < groups.length; i++) {
- super.addCommandArguments("--group-add " + groups[i]);
- }
+ super.addCommandArguments("group-add", String.join(",", groups));
return this;
}
public DockerRunCommand setOverrideCommandWithArgs(
List<String> overrideCommandWithArgs) {
- this.overrrideCommandWithArgs = overrideCommandWithArgs;
+ for(String override: overrideCommandWithArgs) {
+ super.addCommandArguments("launch-command", override);
+ }
return this;
}
@Override
- public String getCommandWithArguments() {
- List<String> argList = new ArrayList<>();
-
- argList.add(super.getCommandWithArguments());
- argList.add(image);
-
- if (overrrideCommandWithArgs != null) {
- argList.addAll(overrrideCommandWithArgs);
- }
-
- return StringUtils.join(" ", argList);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return super.getDockerCommandWithArguments();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
index e9d6c43..ccfff82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
@@ -29,11 +29,11 @@ public class DockerStopCommand extends DockerCommand {
public DockerStopCommand(String containerName) {
super(STOP_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
public DockerStopCommand setGracePeriod(int value) {
- super.addCommandArguments("--time=" + Integer.toString(value));
+ super.addCommandArguments("time", Integer.toString(value));
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 12dbc4c..f23cff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -21,6 +21,7 @@
#include "configuration.h"
#include "util.h"
+#include "get_executable.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
@@ -696,3 +697,19 @@ int get_kv_value(const char *input, char *out, size_t out_len) {
return 0;
}
+
+char *get_config_path(const char *argv0) {
+ char *executable_file = get_executable((char *) argv0);
+ if (!executable_file) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
+ return NULL;
+ }
+
+ const char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ if (conf_file == NULL) {
+ fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
+ }
+ return conf_file;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 1ea5561..7fa684e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -23,10 +23,21 @@
#define _WITH_GETLINE
#endif
-#include <stddef.h>
+#include "config.h"
+
+#define CONF_FILENAME "container-executor.cfg"
+
+// When building as part of a Maven build this value gets defined by using
+// container-executor.conf.dir property. See:
+// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+// for details.
+// NOTE: if this ends up being a relative path it gets resolved relative to
+// the location of the container-executor binary itself, not getwd(3)
+#ifndef HADOOP_CONF_DIR
+#error HADOOP_CONF_DIR must be defined
+#endif
-/** Define a platform-independent constant instead of using PATH_MAX */
-#define EXECUTOR_PATH_MAX 4096
+#include <stddef.h>
// Configuration data structures.
struct kv_pair {
@@ -207,4 +218,6 @@ int get_kv_key(const char *input, char *out, size_t out_len);
*/
int get_kv_value(const char *input, char *out, size_t out_len);
+char *get_config_path(const char* argv0);
+
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index e8bf564..08d69a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -18,7 +18,7 @@
#include "configuration.h"
#include "container-executor.h"
-#include "utils/string-utils.h"
+#include "utils/docker-util.h"
#include "util.h"
#include "config.h"
@@ -43,7 +43,6 @@
#include <sys/mount.h>
#include <sys/wait.h>
#include <getopt.h>
-#include <regex.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -81,11 +80,6 @@ static const char* TC_READ_STATS_OPTS [] = { "-s", "-b", NULL};
//struct to store the user details
struct passwd *user_detail = NULL;
-//Docker container related constants.
-static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_";
-static const char* DOCKER_CLIENT_CONFIG_ARG = "--config=";
-static const char* DOCKER_PULL_COMMAND = "pull";
-
FILE* LOGFILE = NULL;
FILE* ERRORFILE = NULL;
@@ -465,8 +459,9 @@ int is_feature_enabled(const char* feature_key, int default_value,
}
int is_docker_support_enabled() {
- return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
- DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg);
+ return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
+ DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg)
+ || docker_module_enabled(&CFG);
}
int is_tc_support_enabled() {
@@ -474,13 +469,6 @@ int is_tc_support_enabled() {
DEFAULT_TC_SUPPORT_ENABLED, &executor_cfg);
}
-char* check_docker_binary(char *docker_binary) {
- if (docker_binary == NULL) {
- return "docker";
- }
- return docker_binary;
-}
-
/**
* Utility function to concatenate argB to argA using the concat_pattern.
*/
@@ -1157,273 +1145,28 @@ int initialize_app(const char *user, const char *app_id,
return -1;
}
-static char* escape_single_quote(const char *str) {
- int p = 0;
- int i = 0;
- char replacement[] = "'\"'\"'";
- size_t replacement_length = strlen(replacement);
- size_t ret_size = strlen(str) * replacement_length + 1;
- char *ret = (char *) calloc(ret_size, sizeof(char));
- if(ret == NULL) {
- exit(OUT_OF_MEMORY);
- }
- while(str[p] != '\0') {
- if(str[p] == '\'') {
- strncat(ret, replacement, ret_size - strlen(ret));
- i += replacement_length;
- }
- else {
- ret[i] = str[p];
- ret[i + 1] = '\0';
- i++;
- }
- p++;
- }
- return ret;
-}
-
-static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
- char *tmp = escape_single_quote(arg);
- strcat(*str, param);
- strcat(*str, "'");
- if(strlen(*str) + strlen(tmp) > *size) {
- *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024);
- if(*str == NULL) {
- exit(OUT_OF_MEMORY);
- }
- *size = strlen(*str) + strlen(tmp) + 1024;
- }
- strcat(*str, tmp);
- strcat(*str, "' ");
- free(tmp);
-}
-
-char** tokenize_docker_command(const char *input, int *split_counter) {
- char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
- char **linesplit = (char **) malloc(sizeof(char *));
- char *p = NULL;
- *split_counter = 0;
- strncpy(line, input, strlen(input));
-
- p = strtok(line, " ");
- while(p != NULL) {
- linesplit[*split_counter] = p;
- (*split_counter)++;
- linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1)));
- if(linesplit == NULL) {
- fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s",
- strerror(errno));
- fflush(ERRORFILE);
- exit(OUT_OF_MEMORY);
- }
- p = strtok(NULL, " ");
- }
- linesplit[*split_counter] = NULL;
- return linesplit;
-}
-
-int execute_regex_match(const char *regex_str, const char *input) {
- regex_t regex;
- int regex_match;
- if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
- fprintf(LOGFILE, "Unable to compile regex.");
- fflush(LOGFILE);
- exit(ERROR_COMPILING_REGEX);
- }
- regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
- regfree(®ex);
- if(0 == regex_match) {
- return 0;
- }
- return 1;
-}
-
-int validate_docker_image_name(const char *image_name) {
- char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
- return execute_regex_match(regex_str, image_name);
-}
-
-char* sanitize_docker_command(const char *line) {
- static struct option long_options[] = {
- {"name", required_argument, 0, 'n' },
- {"user", required_argument, 0, 'u' },
- {"rm", no_argument, 0, 'r' },
- {"workdir", required_argument, 0, 'w' },
- {"net", required_argument, 0, 'e' },
- {"hostname", required_argument, 0, 'h' },
- {"cgroup-parent", required_argument, 0, 'g' },
- {"privileged", no_argument, 0, 'p' },
- {"cap-add", required_argument, 0, 'a' },
- {"cap-drop", required_argument, 0, 'o' },
- {"device", required_argument, 0, 'i' },
- {"detach", required_argument, 0, 't' },
- {"format", required_argument, 0, 'f' },
- {"group-add", required_argument, 0, 'x' },
- {0, 0, 0, 0}
- };
-
- int c = 0;
- int option_index = 0;
- char *output = NULL;
- size_t output_size = 0;
- char **linesplit;
- int split_counter = 0;
- int len = strlen(line);
-
- linesplit = tokenize_docker_command(line, &split_counter);
-
- output_size = len * 2;
- output = (char *) calloc(output_size, sizeof(char));
- if(output == NULL) {
- exit(OUT_OF_MEMORY);
- }
-
- // Handle docker client config option.
- if(0 == strncmp(linesplit[0], DOCKER_CLIENT_CONFIG_ARG, strlen(DOCKER_CLIENT_CONFIG_ARG))) {
- strcat(output, linesplit[0]);
- strcat(output, " ");
- long index = 0;
- while(index < split_counter) {
- linesplit[index] = linesplit[index + 1];
- if (linesplit[index] == NULL) {
- split_counter--;
- break;
- }
- index++;
- }
- }
-
- // Handle docker pull and image name validation.
- if (0 == strncmp(linesplit[0], DOCKER_PULL_COMMAND, strlen(DOCKER_PULL_COMMAND))) {
- if (0 != validate_docker_image_name(linesplit[1])) {
- fprintf(ERRORFILE, "Invalid Docker image name, exiting.");
- fflush(ERRORFILE);
- exit(DOCKER_IMAGE_INVALID);
- }
- strcat(output, linesplit[0]);
- strcat(output, " ");
- strcat(output, linesplit[1]);
- return output;
- }
-
- strcat(output, linesplit[0]);
- strcat(output, " ");
- optind = 1;
- while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) {
- switch(c) {
- case 'n':
- quote_and_append_arg(&output, &output_size, "--name=", optarg);
- break;
- case 'w':
- quote_and_append_arg(&output, &output_size, "--workdir=", optarg);
- break;
- case 'u':
- quote_and_append_arg(&output, &output_size, "--user=", optarg);
- break;
- case 'e':
- quote_and_append_arg(&output, &output_size, "--net=", optarg);
- break;
- case 'h':
- quote_and_append_arg(&output, &output_size, "--hostname=", optarg);
- break;
- case 'v':
- quote_and_append_arg(&output, &output_size, "-v ", optarg);
- break;
- case 'a':
- quote_and_append_arg(&output, &output_size, "--cap-add=", optarg);
- break;
- case 'o':
- quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg);
- break;
- case 'd':
- strcat(output, "-d ");
- break;
- case 'r':
- strcat(output, "--rm ");
- break;
- case 'g':
- quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg);
- break;
- case 'p':
- strcat(output, "--privileged ");
- break;
- case 'i':
- quote_and_append_arg(&output, &output_size, "--device=", optarg);
- break;
- case 't':
- quote_and_append_arg(&output, &output_size, "--detach=", optarg);
- break;
- case 'f':
- strcat(output, "--format=");
- strcat(output, optarg);
- strcat(output, " ");
- break;
- case 'x':
- quote_and_append_arg(&output, &output_size, "--group-add ", optarg);
- break;
- default:
- fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
- fflush(LOGFILE);
- return NULL;
- break;
- }
- }
-
- if(optind < split_counter) {
- while(optind < split_counter) {
- if (0 == strncmp(linesplit[optind], DOCKER_CONTAINER_NAME_PREFIX, strlen(DOCKER_CONTAINER_NAME_PREFIX))) {
- if (1 != validate_container_id(linesplit[optind])) {
- fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", linesplit[optind]);
- fflush(ERRORFILE);
- exit(DOCKER_CONTAINER_NAME_INVALID);
- }
- strcat(output, linesplit[optind++]);
- } else {
- quote_and_append_arg(&output, &output_size, "", linesplit[optind++]);
- }
- }
- }
-
- return output;
-}
-
-char* parse_docker_command_file(const char* command_file) {
- size_t len = 0;
- char *line = NULL;
- ssize_t read;
- FILE *stream;
- stream = fopen(command_file, "r");
- if (stream == NULL) {
- fprintf(ERRORFILE, "Cannot open file %s - %s",
- command_file, strerror(errno));
- fflush(ERRORFILE);
- exit(ERROR_OPENING_DOCKER_FILE);
- }
- if ((read = getline(&line, &len, stream)) == -1) {
- fprintf(ERRORFILE, "Error reading command_file %s\n", command_file);
- fflush(ERRORFILE);
- exit(ERROR_READING_DOCKER_FILE);
- }
- fclose(stream);
-
- char* ret = sanitize_docker_command(line);
- if(ret == NULL) {
- exit(ERROR_SANITIZING_DOCKER_COMMAND);
+char *construct_docker_command(const char *command_file) {
+ int ret = 0;
+ size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
+ char *buffer = alloc_and_clear_memory(command_size, sizeof(char));
+ ret = get_docker_command(command_file, &CFG, buffer, command_size);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Error constructing docker command, docker error code=%d, error message='%s'\n", ret,
+ get_docker_error_message(ret));
+ fflush(ERRORFILE);
+ exit(DOCKER_RUN_FAILED);
}
- fprintf(ERRORFILE, "Using command %s\n", ret);
- fflush(ERRORFILE);
-
- return ret;
+ return buffer;
}
int run_docker(const char *command_file) {
- char* docker_command = parse_docker_command_file(command_file);
- char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char* docker_command = construct_docker_command(command_file);
+ char* docker_binary = get_docker_binary(&CFG);
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- char* docker_command_with_binary = calloc(sizeof(char), command_size);
+ char* docker_command_with_binary = alloc_and_clear_memory(command_size, sizeof(char));
snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
+ fprintf(LOGFILE, "Invoking '%s'\n", docker_command_with_binary);
char **args = split_delimiter(docker_command_with_binary, " ");
int exit_code = -1;
@@ -1437,8 +1180,9 @@ int run_docker(const char *command_file) {
free(docker_command_with_binary);
free(docker_command);
exit_code = DOCKER_RUN_FAILED;
+ } else {
+ exit_code = 0;
}
- exit_code = 0;
return exit_code;
}
@@ -1583,18 +1327,17 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- docker_command_with_binary = calloc(sizeof(char), command_size);
- docker_wait_command = calloc(sizeof(char), command_size);
- docker_logs_command = calloc(sizeof(char), command_size);
- docker_inspect_command = calloc(sizeof(char), command_size);
- docker_rm_command = calloc(sizeof(char), command_size);
+ docker_command_with_binary = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_wait_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_logs_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_inspect_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_rm_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
gid_t user_gid = getegid();
uid_t prev_uid = geteuid();
- char *docker_command = parse_docker_command_file(command_file);
- char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char *docker_command = NULL;
+ char *docker_binary = NULL;
fprintf(LOGFILE, "Creating script paths...\n");
exit_code = create_script_paths(
@@ -1617,6 +1360,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup;
}
+ docker_command = construct_docker_command(command_file);
+ docker_binary = get_docker_binary(&CFG);
+
fprintf(LOGFILE, "Getting exit code file...\n");
exit_code_file = get_exit_code_file(pid_file);
if (NULL == exit_code_file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index aa38abf..6d4220f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -59,16 +59,12 @@ enum operations {
#define MIN_USERID_KEY "min.user.id"
#define BANNED_USERS_KEY "banned.users"
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
-#define DOCKER_BINARY_KEY "docker.binary"
#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
#define TMP_DIR "tmp"
extern struct passwd *user_detail;
-// get the executable's filename
-char* get_executable(char *argv0);
-
//function used to load the configurations present in the secure config
void read_executor_config(const char* file_name);
@@ -258,11 +254,6 @@ int is_docker_support_enabled();
*/
int run_docker(const char *command_file);
-/**
- * Sanitize docker commands. Returns NULL if there was any failure.
-*/
-char* sanitize_docker_command(const char *line);
-
/*
* Compile the regex_str and determine if the input string matches.
* Return 0 on match, 1 of non-match.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index 55973a2..e1ec293 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -29,12 +29,9 @@
*/
#include "config.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "util.h"
#include <errno.h>
-#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
new file mode 100644
index 0000000..ee6b375
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+
+/**
+ * Get the path to executable that is currently running
+ * @param argv0 the name of the executable
+ * @return the path to the currently running executable
+ */
+char* get_executable(char *argv0);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index a05dc78..7b8f63f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -20,30 +20,15 @@
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
+#include "get_executable.h"
#include "modules/gpu/gpu-module.h"
#include "modules/cgroups/cgroups-operations.h"
#include <errno.h>
#include <grp.h>
-#include <limits.h>
#include <unistd.h>
-#include <signal.h>
-#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <sys/stat.h>
-
-#define CONF_FILENAME "container-executor.cfg"
-
-// When building as part of a Maven build this value gets defined by using
-// container-executor.conf.dir property. See:
-// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
-// for details.
-// NOTE: if this ends up being a relative path it gets resolved relative to
-// the location of the container-executor binary itself, not getwd(3)
-#ifndef HADOOP_CONF_DIR
- #error HADOOP_CONF_DIR must be defined
-#endif
static void display_usage(FILE *stream) {
fprintf(stream,
@@ -145,25 +130,21 @@ of whether an explicit checksetup operation is requested. */
static void assert_valid_setup(char *argv0) {
int ret;
char *executable_file = get_executable(argv0);
- if (!executable_file) {
- fprintf(ERRORFILE,"realpath of executable: %s\n",
- errno != 0 ? strerror(errno) : "unknown");
+ if (!executable_file || executable_file[0] == 0) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
flush_and_close_log_files();
- exit(-1);
+ exit(INVALID_CONFIG_FILE);
}
- char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
- char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ char *conf_file = get_config_path(argv0);
if (conf_file == NULL) {
- free(executable_file);
- fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
if (check_configuration_permissions(conf_file) != 0) {
- free(executable_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
index f0c6d16..4c76e51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
@@ -16,9 +16,8 @@
* limitations under the License.
*/
+#include "module-configs.h"
#include "util.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "modules/common/constants.h"
#include <string.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
index d58c618..b8c57df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
@@ -23,6 +23,7 @@
#ifndef _MODULES_COMMON_MODULE_CONFIGS_H_
#define _MODULES_COMMON_MODULE_CONFIGS_H_
+#include "configuration.h"
/**
* check if module enabled given name of module.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
index 8e39ca8..a9539cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
@@ -17,10 +17,10 @@
*/
#include "util.h"
-#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
+#include <regex.h>
char** split_delimiter(char *value, const char *delim) {
char **return_values = NULL;
@@ -132,3 +132,61 @@ char* trim(const char* input) {
ret[val_end - val_begin] = '\0';
return ret;
}
+
+int execute_regex_match(const char *regex_str, const char *input) {
+ regex_t regex;
+ int regex_match;
+ if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
+ fprintf(LOGFILE, "Unable to compile regex.");
+ fflush(LOGFILE);
+ exit(ERROR_COMPILING_REGEX);
+ }
+ regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
+ regfree(®ex);
+ if(0 == regex_match) {
+ return 0;
+ }
+ return 1;
+}
+
+char* escape_single_quote(const char *str) {
+ int p = 0;
+ int i = 0;
+ char replacement[] = "'\"'\"'";
+ size_t replacement_length = strlen(replacement);
+ size_t ret_size = strlen(str) * replacement_length + 1;
+ char *ret = (char *) alloc_and_clear_memory(ret_size, sizeof(char));
+ if(ret == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ while(str[p] != '\0') {
+ if(str[p] == '\'') {
+ strncat(ret, replacement, ret_size - strlen(ret));
+ i += replacement_length;
+ }
+ else {
+ ret[i] = str[p];
+ i++;
+ }
+ p++;
+ }
+ ret[i] = '\0';
+ return ret;
+}
+
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
+ char *tmp = escape_single_quote(arg);
+ int alloc_block = 1024;
+ strcat(*str, param);
+ strcat(*str, "'");
+ if (strlen(*str) + strlen(tmp) > *size) {
+ *size = (strlen(*str) + strlen(tmp) + alloc_block) * sizeof(char);
+ *str = (char *) realloc(*str, *size);
+ if (*str == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ }
+ strcat(*str, tmp);
+ strcat(*str, "' ");
+ free(tmp);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index fa21def..8758d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -19,7 +19,11 @@
#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
+/** Define a platform-independent constant instead of using PATH_MAX, set to 4K */
+#define EXECUTOR_PATH_MAX 4096
+
#include <stdio.h>
+#include <stdlib.h>
enum errorcodes {
INVALID_ARGUMENT_NUMBER = 1,
@@ -62,7 +66,7 @@ enum errorcodes {
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
ERROR_SANITIZING_DOCKER_COMMAND = 39,
DOCKER_IMAGE_INVALID = 40,
- DOCKER_CONTAINER_NAME_INVALID = 41,
+ // DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED)
ERROR_COMPILING_REGEX = 42
};
@@ -119,4 +123,44 @@ void free_values(char **values);
*/
char* trim(const char *input);
+/**
+ * Run a regex to check if the provided input matches against it
+ * @param regex_str Regex to run
+ * @param input String to run the regex against
+ * @return 0 on match, non-zero on no match
+ */
+int execute_regex_match(const char *regex_str, const char *input);
+
+/**
+ * Helper function to escape single-quotes in a string. The assumption is that the string passed will be enclosed in
+ * single quotes and passed to bash for a command invocation.
+ * @param str The string in which to esacpe single quotes
+ * @return String with single quotes escaped, must be freed by the user.
+ */
+char* escape_single_quote(const char *str);
+
+/**
+ * Helper function to quote the argument to a parameter and then append it to the provided string.
+ * @param str Buffer to which the param='arg' string must be appended
+ * @param size Size of the buffer
+ * @param param Param name
+ * @param arg Argument to be quoted
+ */
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg);
+
+/**
+ * Helper function to allocate and clear a block of memory. It'll call exit if the allocation fails.
+ * @param num Num of elements to be allocated
+ * @param size Size of each element
+ * @return Pointer to the allocated memory, must be freed by the user
+ */
+inline void* alloc_and_clear_memory(size_t num, size_t size) {
+ void *ret = calloc(num, size);
+ if (ret == NULL) {
+ printf("Could not allocate memory, exiting\n");
+ exit(OUT_OF_MEMORY);
+ }
+ return ret;
+}
+
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org