You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/09/22 22:07:51 UTC
[01/13] hadoop git commit: YARN-6991. "Kill application" button does
not show error if other user tries to kill the application for secure
cluster. (Suma Shivaprasad via wangda)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 0880d5e98 -> 16dd69ad4
YARN-6991. "Kill application" button does not show error if other user tries to kill the application for secure cluster. (Suma Shivaprasad via wangda)
Change-Id: I7e7894b24609709f89064ee5882f055dbb09080b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/263e2c69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/263e2c69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/263e2c69
Branch: refs/heads/HDFS-7240
Commit: 263e2c692a4b0013766cd1f6b6d7ed674b2b1040
Parents: b9e423f
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Sep 21 12:00:53 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Sep 21 12:00:53 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/webapp/AppBlock.java | 44 +++++++++++---------
1 file changed, 24 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/263e2c69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index 08e75ac..b429b5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -137,6 +137,30 @@ public class AppBlock extends HtmlBlock {
setTitle(join("Application ", aid));
+ //Validate if able to read application attempts
+ // which should also validate if kill is allowed for the user based on ACLs
+
+ Collection<ApplicationAttemptReport> attempts;
+ try {
+ final GetApplicationAttemptsRequest request =
+ GetApplicationAttemptsRequest.newInstance(appID);
+ attempts = callerUGI.doAs(
+ new PrivilegedExceptionAction<Collection<
+ ApplicationAttemptReport>>() {
+ @Override
+ public Collection<ApplicationAttemptReport> run() throws Exception {
+ return getApplicationAttemptsReport(request);
+ }
+ });
+ } catch (Exception e) {
+ String message =
+ "Failed to read the attempts of the application " + appID + ".";
+ LOG.error(message, e);
+ html.p().__(message).__();
+ return;
+ }
+
+
// YARN-6890. for secured cluster allow anonymous UI access, application kill
// shouldn't be there.
boolean unsecuredUIForSecuredCluster = UserGroupInformation.isSecurityEnabled()
@@ -183,26 +207,6 @@ public class AppBlock extends HtmlBlock {
generateOverviewTable(app, schedulerPath, webUiType, appReport);
- Collection<ApplicationAttemptReport> attempts;
- try {
- final GetApplicationAttemptsRequest request =
- GetApplicationAttemptsRequest.newInstance(appID);
- attempts = callerUGI.doAs(
- new PrivilegedExceptionAction<Collection<
- ApplicationAttemptReport>>() {
- @Override
- public Collection<ApplicationAttemptReport> run() throws Exception {
- return getApplicationAttemptsReport(request);
- }
- });
- } catch (Exception e) {
- String message =
- "Failed to read the attempts of the application " + appID + ".";
- LOG.error(message, e);
- html.p().__(message).__();
- return;
- }
-
createApplicationMetricsTable(html);
html.__(InfoBlock.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/13] hadoop git commit: HDFS-12507. StripedBlockUtil.java:694:
warning - Tag @link: reference not found: StripingCell. Contributed by Mukul
Kumar Singh
Posted by ae...@apache.org.
HDFS-12507. StripedBlockUtil.java:694: warning - Tag @link: reference not found: StripingCell. Contributed by Mukul Kumar Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3baae432
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3baae432
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3baae432
Branch: refs/heads/HDFS-7240
Commit: 3baae4322e3f2035c0a99eb9e4306567883581d1
Parents: bfd1a72
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Fri Sep 22 10:37:04 2017 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Fri Sep 22 10:37:04 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3baae432/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 896ebc6..9e24576 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -575,7 +575,7 @@ public class StripedBlockUtil {
* TODO: consider parity cells
*/
@VisibleForTesting
- static class StripingCell {
+ public static class StripingCell {
final ErasureCodingPolicy ecPolicy;
/** Logical order in a block group, used when doing I/O to a block group. */
final int idxInBlkGroup;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/13] hadoop git commit: MAPREDUCE-6965. QuasiMonteCarlo should use
Time.monotonicNow for measuring durations. Contributed by Chetna Chaudhari
Posted by ae...@apache.org.
MAPREDUCE-6965. QuasiMonteCarlo should use Time.monotonicNow for measuring durations. Contributed by Chetna Chaudhari
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/908d8e91
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/908d8e91
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/908d8e91
Branch: refs/heads/HDFS-7240
Commit: 908d8e914ef55aca69cb8e725e62bda29d1073cc
Parents: c71d137
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Sep 22 09:34:53 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Sep 22 09:34:53 2017 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/908d8e91/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
index 3048fd3..25a25bf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -302,13 +303,13 @@ public class QuasiMonteCarlo extends Configured implements Tool {
//start a map/reduce job
System.out.println("Starting Job");
- final long startTime = System.currentTimeMillis();
+ final long startTime = Time.monotonicNow();
job.waitForCompletion(true);
if (!job.isSuccessful()) {
System.out.println("Job " + job.getJobID() + " failed!");
System.exit(1);
}
- final double duration = (System.currentTimeMillis() - startTime)/1000.0;
+ final double duration = (Time.monotonicNow() - startTime)/1000.0;
System.out.println("Job Finished in " + duration + " seconds");
//read outputs
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/13] hadoop git commit: YARN-7118. AHS REST API can return
NullPointerException. Contributed by Billie Rinaldi.
Posted by ae...@apache.org.
YARN-7118. AHS REST API can return NullPointerException. Contributed by Billie Rinaldi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4002bf0a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4002bf0a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4002bf0a
Branch: refs/heads/HDFS-7240
Commit: 4002bf0a9e7e3619a3bdcff071f0369cbf2873ad
Parents: b133dc5
Author: Junping Du <ju...@apache.org>
Authored: Fri Sep 22 13:58:09 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Fri Sep 22 13:58:09 2017 -0700
----------------------------------------------------------------------
...pplicationHistoryManagerOnTimelineStore.java | 16 ++++++++---
.../webapp/TestAHSWebServices.java | 28 +++++++++++++++++---
.../hadoop/yarn/server/webapp/WebServices.java | 3 ++-
3 files changed, 38 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4002bf0a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 9600251..ecaaf1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -146,7 +146,11 @@ public class TestApplicationHistoryManagerOnTimelineStore {
} else if (i == 3) {
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED,
- true));
+ true, false));
+ } else if (i == SCALE + 1) {
+ entities.addEntity(createApplicationTimelineEntity(
+ appId, false, false, false, false, YarnApplicationState.FINISHED,
+ false, true));
} else {
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED));
@@ -497,13 +501,14 @@ public class TestApplicationHistoryManagerOnTimelineStore {
boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state) {
return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId,
- wrongAppId, enableUpdateEvent, state, false);
+ wrongAppId, enableUpdateEvent, state, false, false);
}
private static TimelineEntity createApplicationTimelineEntity(
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId, boolean enableUpdateEvent,
- YarnApplicationState state, boolean missingPreemptMetrics) {
+ YarnApplicationState state, boolean missingPreemptMetrics,
+ boolean missingQueue) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
if (wrongAppId) {
@@ -519,7 +524,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
"test app type");
entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1");
- entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
+ if (!missingQueue) {
+ entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
+ "test queue");
+ }
entityInfo.put(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4002bf0a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index bca5d5c..154c68a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -100,7 +100,7 @@ public class TestAHSWebServices extends JerseyTestBase {
private static ApplicationHistoryClientService historyClientService;
private static AHSWebServices ahsWebservice;
private static final String[] USERS = new String[] { "foo" , "bar" };
- private static final int MAX_APPS = 5;
+ private static final int MAX_APPS = 6;
private static Configuration conf;
private static FileSystem fs;
private static final String remoteLogRootDir = "target/logs/";
@@ -364,7 +364,27 @@ public class TestAHSWebServices extends JerseyTestBase {
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
- assertEquals("incorrect number of elements", 5, array.length());
+ assertEquals("incorrect number of elements", MAX_APPS, array.length());
+ }
+
+ @Test
+ public void testQueueQuery() throws Exception {
+ WebResource r = resource();
+ ClientResponse response =
+ r.path("ws").path("v1").path("applicationhistory").path("apps")
+ .queryParam("queue", "test queue")
+ .queryParam("user.name", USERS[round])
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertResponseStatusCode(Status.OK, response.getStatusInfo());
+ assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ JSONArray array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", MAX_APPS - 1,
+ array.length());
}
@Test
@@ -414,7 +434,7 @@ public class TestAHSWebServices extends JerseyTestBase {
JSONObject appAttempts = json.getJSONObject("appAttempts");
assertEquals("incorrect number of elements", 1, appAttempts.length());
JSONArray array = appAttempts.getJSONArray("appAttempt");
- assertEquals("incorrect number of elements", 5, array.length());
+ assertEquals("incorrect number of elements", MAX_APPS, array.length());
}
@Test
@@ -471,7 +491,7 @@ public class TestAHSWebServices extends JerseyTestBase {
JSONObject containers = json.getJSONObject("containers");
assertEquals("incorrect number of elements", 1, containers.length());
JSONArray array = containers.getJSONArray("container");
- assertEquals("incorrect number of elements", 5, array.length());
+ assertEquals("incorrect number of elements", MAX_APPS, array.length());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4002bf0a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 6bb6c98..1399099 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -192,7 +192,8 @@ public class WebServices {
}
}
if (queueQuery != null && !queueQuery.isEmpty()) {
- if (!appReport.getQueue().equals(queueQuery)) {
+ if (appReport.getQueue() == null || !appReport.getQueue()
+ .equals(queueQuery)) {
continue;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/13] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16dd69ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16dd69ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16dd69ad
Branch: refs/heads/HDFS-7240
Commit: 16dd69ad4bb8b57a6aa15260e31700a527cef4e5
Parents: 0880d5e 8d29bf5
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Sep 22 14:48:33 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Sep 22 14:48:33 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/util/StripedBlockUtil.java | 2 +-
.../OfflineImageViewerPB.java | 11 +--
.../hadoop-mapreduce-examples/pom.xml | 4 +
.../hadoop/examples/BaileyBorweinPlouffe.java | 7 +-
.../apache/hadoop/examples/DBCountPageView.java | 7 +-
.../apache/hadoop/examples/QuasiMonteCarlo.java | 5 +-
.../hadoop/examples/dancing/DancingLinks.java | 7 +-
.../org/apache/hadoop/examples/pi/DistSum.java | 11 +--
.../hadoop/examples/terasort/TeraGen.java | 6 +-
.../examples/terasort/TeraOutputFormat.java | 7 +-
.../hadoop/examples/terasort/TeraScheduler.java | 7 +-
.../hadoop/examples/terasort/TeraSort.java | 6 +-
.../hadoop/examples/terasort/TestTeraSort.java | 6 +-
hadoop-project/pom.xml | 2 +-
.../apache/hadoop/mapred/gridmix/SleepJob.java | 5 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 21 +++++
.../src/main/resources/yarn-default.xml | 19 +++++
...pplicationHistoryManagerOnTimelineStore.java | 16 +++-
.../webapp/TestAHSWebServices.java | 28 ++++++-
.../hadoop/yarn/server/webapp/AppBlock.java | 44 +++++-----
.../hadoop/yarn/server/webapp/WebServices.java | 3 +-
.../privileged/PrivilegedOperationExecutor.java | 2 +-
.../runtime/DefaultLinuxContainerRuntime.java | 8 +-
.../runtime/DockerLinuxContainerRuntime.java | 85 ++++++++++++++++++--
.../linux/runtime/docker/DockerRunCommand.java | 7 ++
.../impl/container-executor.c | 4 +
.../TestLinuxContainerExecutorWithMocks.java | 67 ++++++++++++++-
.../runtime/TestDockerContainerRuntime.java | 82 ++++++++++++++++++-
.../scheduler/fair/FSLeafQueue.java | 10 ---
.../scheduler/fair/TestFSLeafQueue.java | 6 +-
.../scheduler/fair/TestFairScheduler.java | 6 +-
31 files changed, 397 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16dd69ad/hadoop-project/pom.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/13] hadoop git commit: YARN-7034. DefaultLinuxContainerRuntime
and DockerLinuxContainerRuntime sends client environment variables to
container-executor. Contributed by Miklos Szegedi.
Posted by ae...@apache.org.
YARN-7034. DefaultLinuxContainerRuntime and DockerLinuxContainerRuntime sends client environment variables to container-executor. Contributed by Miklos Szegedi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5e1851d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5e1851d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5e1851d
Branch: refs/heads/HDFS-7240
Commit: e5e1851d803bf8d8b96fec1b5c0058014e9329d0
Parents: a92ef03
Author: Junping Du <ju...@apache.org>
Authored: Thu Sep 21 14:01:16 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Thu Sep 21 14:01:16 2017 -0700
----------------------------------------------------------------------
.../privileged/PrivilegedOperationExecutor.java | 2 +-
.../runtime/DefaultLinuxContainerRuntime.java | 8 +--
.../runtime/DockerLinuxContainerRuntime.java | 9 +--
.../TestLinuxContainerExecutorWithMocks.java | 67 +++++++++++++++++++-
.../runtime/TestDockerContainerRuntime.java | 3 +-
5 files changed, 73 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
index 5a3ce74..9f13a5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
@@ -204,7 +204,7 @@ public class PrivilegedOperationExecutor {
public String executePrivilegedOperation(PrivilegedOperation operation,
boolean grabOutput) throws PrivilegedOperationException {
return executePrivilegedOperation(null, operation, null, null, grabOutput,
- true);
+ false);
}
//Utility functions for squashing together operations in supported ways
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index d09e4a1..6c3ae85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -80,7 +80,6 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void launchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
PrivilegedOperation launchOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.LAUNCH_CONTAINER);
@@ -116,8 +115,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
try {
privilegedOperationExecutor.executePrivilegedOperation(prefixCommands,
- launchOp, null, container.getLaunchContext().getEnvironment(),
- false, false);
+ launchOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
@@ -129,7 +127,6 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void signalContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
PrivilegedOperation signalOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.SIGNAL_CONTAINER);
@@ -148,8 +145,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
.getInstance(conf);
executor.executePrivilegedOperation(null,
- signalOp, null, container.getLaunchContext().getEnvironment(),
- false, true);
+ signalOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
//Don't log the failure here. Some kinds of signaling failures are
// acceptable. Let the calling executor decide what to do.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 1ccd365..6491a99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -549,8 +549,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
try {
privilegedOperationExecutor.executePrivilegedOperation(null,
- launchOp, null, container.getLaunchContext().getEnvironment(),
- false, false);
+ launchOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
@@ -563,7 +562,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void signalContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL);
PrivilegedOperation privOp = null;
@@ -594,8 +592,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
try {
privilegedOperationExecutor.executePrivilegedOperation(null,
- privOp, null, container.getLaunchContext().getEnvironment(),
- false, false);
+ privOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
throw new ContainerExecutionException("Signal container failed", e
.getExitCode(), e.getOutput(), e.getErrorOutput());
@@ -623,7 +620,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
privOp.appendArgs(commandFile);
String output = privilegedOperationExecutor
.executePrivilegedOperation(null, privOp, null,
- container.getLaunchContext().getEnvironment(), true, false);
+ null, true, false);
LOG.info("Docker inspect output for " + containerId + ": " + output);
int index = output.lastIndexOf(',');
if (index == -1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 6ca96db..9fa22d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -72,10 +72,16 @@ import org.junit.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static org.mockito.Matchers.anyListOf;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
public class TestLinuxContainerExecutorWithMocks {
@@ -91,6 +97,8 @@ public class TestLinuxContainerExecutorWithMocks {
private String tmpMockExecutor;
private LinuxContainerExecutor mockExec = null;
+ private LinuxContainerExecutor mockExecMockRuntime = null;
+ private PrivilegedOperationExecutor mockPrivilegedExec;
private final File mockParamFile = new File("./params.txt");
private LocalDirsHandlerService dirsHandler;
@@ -140,22 +148,28 @@ public class TestLinuxContainerExecutorWithMocks {
Configuration conf = new YarnConfiguration();
LinuxContainerRuntime linuxContainerRuntime;
+ LinuxContainerRuntime mockLinuxContainerRuntime;
setupMockExecutor(MOCK_EXECUTOR, conf);
linuxContainerRuntime = new DefaultLinuxContainerRuntime(
PrivilegedOperationExecutor.getInstance(conf));
+ mockPrivilegedExec = Mockito.mock(PrivilegedOperationExecutor.class);
+ mockLinuxContainerRuntime = new DefaultLinuxContainerRuntime(
+ mockPrivilegedExec);
dirsHandler = new LocalDirsHandlerService();
dirsHandler.init(conf);
linuxContainerRuntime.initialize(conf);
mockExec = new LinuxContainerExecutor(linuxContainerRuntime);
mockExec.setConf(conf);
+ mockExecMockRuntime = new LinuxContainerExecutor(mockLinuxContainerRuntime);
+ mockExecMockRuntime.setConf(conf);
}
@After
public void tearDown() {
deleteMockParamFile();
}
-
+
@Test
public void testContainerLaunch()
throws IOException, ConfigurationException {
@@ -168,7 +182,7 @@ public class TestLinuxContainerExecutorWithMocks {
ContainerId cId = mock(ContainerId.class);
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
HashMap<String, String> env = new HashMap<String,String>();
-
+
when(container.getContainerId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
@@ -605,4 +619,53 @@ public class TestLinuxContainerExecutorWithMocks {
e.getMessage().contains("exit code"));
}
}
+
+ @Test
+ public void testContainerLaunchEnvironment()
+ throws IOException, ConfigurationException,
+ PrivilegedOperationException {
+ String appSubmitter = "nobody";
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+ Container container = mock(Container.class);
+ ContainerId cId = mock(ContainerId.class);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap<String, String> env = new HashMap<String, String>();
+ env.put("FROM_CLIENT", "1");
+
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+
+ when(cId.toString()).thenReturn(containerId);
+
+ when(context.getEnvironment()).thenReturn(env);
+
+ Path scriptPath = new Path("file:///bin/echo");
+ Path tokensPath = new Path("file:///dev/null");
+ Path workDir = new Path("/tmp");
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ mockExecMockRuntime.activateContainer(cId, pidFile);
+ mockExecMockRuntime.launchContainer(new ContainerStartContext.Builder()
+ .setContainer(container)
+ .setNmPrivateContainerScriptPath(scriptPath)
+ .setNmPrivateTokensPath(tokensPath)
+ .setUser(appSubmitter)
+ .setAppId(appId)
+ .setContainerWorkDir(workDir)
+ .setLocalDirs(dirsHandler.getLocalDirs())
+ .setLogDirs(dirsHandler.getLogDirs())
+ .setFilecacheDirs(new ArrayList<>())
+ .setUserLocalDirs(new ArrayList<>())
+ .setContainerLocalDirs(new ArrayList<>())
+ .setContainerLogDirs(new ArrayList<>())
+ .build());
+ ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
+ PrivilegedOperation.class);
+ // Verify that
+ verify(mockPrivilegedExec, times(1))
+ .executePrivilegedOperation(anyListOf(
+ String.class), opCaptor.capture(), any(
+ File.class), eq(null), eq(false), eq(false));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 3f4bb5c..0e314bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -116,6 +116,7 @@ public class TestDockerContainerRuntime {
cId = mock(ContainerId.class);
context = mock(ContainerLaunchContext.class);
env = new HashMap<String, String>();
+ env.put("FROM_CLIENT", "1");
image = "busybox:latest";
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE, image);
@@ -204,7 +205,7 @@ public class TestDockerContainerRuntime {
// warning annotation on the entire method
verify(mockExecutor, times(1))
.executePrivilegedOperation(anyList(), opCaptor.capture(), any(
- File.class), any(Map.class), eq(false), eq(false));
+ File.class), eq(null), eq(false), eq(false));
//verification completed. we need to isolate specific invications.
// hence, reset mock here
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/13] hadoop git commit: YARN-4266. Allow users to enter containers
as UID:GID pair instead of by username. Contributed by luhuichun, Zhankun Tang,
and Eric Badger.
Posted by ae...@apache.org.
YARN-4266. Allow users to enter containers as UID:GID pair instead of by username. Contributed by luhuichun, Zhankun Tang, and Eric Badger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfd1a72b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfd1a72b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfd1a72b
Branch: refs/heads/HDFS-7240
Commit: bfd1a72ba8fbb06da73fede2a85e0b544d6ab43f
Parents: e5e1851
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 21 17:41:34 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 21 17:41:34 2017 -0500
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 21 ++++++
.../src/main/resources/yarn-default.xml | 19 +++++
.../runtime/DockerLinuxContainerRuntime.java | 76 ++++++++++++++++++-
.../linux/runtime/docker/DockerRunCommand.java | 7 ++
.../impl/container-executor.c | 4 +
.../runtime/TestDockerContainerRuntime.java | 79 +++++++++++++++++++-
6 files changed, 204 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 114453f..f58833c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1587,6 +1587,27 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS =
false;
+ /** enable user remapping. */
+ public static final String NM_DOCKER_ENABLE_USER_REMAPPING =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "enable-userremapping.allowed";
+
+ /** Set enable user remapping as false by default. */
+ public static final boolean DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING = false;
+
+ /** lower limit for acceptable uids of user remapped user. */
+ public static final String NM_DOCKER_USER_REMAPPING_UID_THRESHOLD =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-uid-threshold";
+
+ /** Set user remapping lower uid limit to 1 by default. */
+ public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD = 1;
+
+ /** lower limit for acceptable gids of user remapped user. */
+ public static final String NM_DOCKER_USER_REMAPPING_GID_THRESHOLD =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-gid-threshold";
+
+ /** Set user remapping lower gid limit to 1 by default. */
+ public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD = 1;
+
/** ACL list for users allowed to run privileged containers. */
public static final String NM_DOCKER_PRIVILEGED_CONTAINERS_ACL =
DOCKER_CONTAINER_RUNTIME_PREFIX + "privileged-containers.acl";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0440458..8453dc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1643,6 +1643,25 @@
</property>
<property>
+ <description>Property to enable docker user remapping</description>
+ <name>yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>lower limit for acceptable uids of user remapped user</description>
+ <name>yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold</name>
+ <value>1</value>
+ </property>
+
+
+ <property>
+ <description>lower limit for acceptable gids of user remapped user</description>
+ <name>yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold</name>
+ <value>1</value>
+ </property>
+
+ <property>
<description>The mode in which the Java Container Sandbox should run detailed by
the JavaSandboxLinuxContainerRuntime.</description>
<name>yarn.nodemanager.runtime.linux.sandbox-mode</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 6491a99..b8d9b0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -164,6 +165,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
"YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
@InterfaceAudience.Private
+ public static final String ENV_DOCKER_CONTAINER_RUN_ENABLE_USER_REMAPPING =
+ "YARN_CONTAINER_RUNTIME_DOCKER_RUN_ENABLE_USER_REMAPPING";
+ @InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
@@ -175,6 +179,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private String cgroupsRootDirectory;
private CGroupsHandler cGroupsHandler;
private AccessControlList privilegedContainersAcl;
+ private boolean enableUserReMapping;
+ private int userRemappingUidThreshold;
+ private int userRemappingGidThreshold;
/**
* Return whether the given environment variables indicate that the operation
@@ -260,6 +267,18 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
privilegedContainersAcl = new AccessControlList(conf.getTrimmed(
YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL,
YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL));
+
+ enableUserReMapping = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING,
+ YarnConfiguration.DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING);
+
+ userRemappingUidThreshold = conf.getInt(
+ YarnConfiguration.NM_DOCKER_USER_REMAPPING_UID_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD);
+
+ userRemappingGidThreshold = conf.getInt(
+ YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD,
+ YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD);
}
@Override
@@ -436,6 +455,34 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
"resource: " + mount);
}
+ private String getUserIdInfo(String userName)
+ throws ContainerExecutionException {
+ String id = "";
+ Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-u", userName});
+ try {
+ shexec.execute();
+ id = shexec.getOutput().replaceAll("[^0-9]", "");
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ }
+ return id;
+ }
+
+ private String[] getGroupIdInfo(String userName)
+ throws ContainerExecutionException {
+ String[] id = null;
+ Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-G", userName});
+ try {
+ shexec.execute();
+ id = shexec.getOutput().replace("\n", "").split(" ");
+ } catch (Exception e) {
+ throw new ContainerExecutionException(e);
+ }
+ return id;
+ }
+
@Override
public void launchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
@@ -458,7 +505,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
String containerIdStr = container.getContainerId().toString();
String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
+ String dockerRunAsUser = runAsUser;
Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
+ String[] groups = null;
+
+ if (enableUserReMapping) {
+ String uid = getUserIdInfo(runAsUser);
+ groups = getGroupIdInfo(runAsUser);
+ String gid = groups[0];
+ if(Integer.parseInt(uid) < userRemappingUidThreshold) {
+ String message = "uid: " + uid + " below threshold: "
+ + userRemappingUidThreshold;
+ throw new ContainerExecutionException(message);
+ }
+ for(int i = 0; i < groups.length; i++) {
+ String group = groups[i];
+ if (Integer.parseInt(group) < userRemappingGidThreshold) {
+ String message = "gid: " + group
+ + " below threshold: " + userRemappingGidThreshold;
+ throw new ContainerExecutionException(message);
+ }
+ }
+ dockerRunAsUser = uid + ":" + gid;
+ }
+
//List<String> -> stored as List -> fetched/converted to List<String>
//we can't do better here thanks to type-erasure
@SuppressWarnings("unchecked")
@@ -481,7 +551,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@SuppressWarnings("unchecked")
DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,
- runAsUser, imageName)
+ dockerRunAsUser, imageName)
.detachOnRun()
.setContainerWorkDir(containerWorkDir.toString())
.setNetworkType(network);
@@ -542,6 +612,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
runCommand.setOverrideCommandWithArgs(overrideCommands);
}
+ if(enableUserReMapping) {
+ runCommand.groupAdd(groups);
+ }
+
String commandFile = dockerClient.writeCommandToTempFile(runCommand,
containerIdStr);
PrivilegedOperation launchOp = buildLaunchOp(ctx,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index b645754..1e1e6e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -114,6 +114,13 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ public DockerRunCommand groupAdd(String[] groups) {
+ for(int i = 0; i < groups.length; i++) {
+ super.addCommandArguments("--group-add " + groups[i]);
+ }
+ return this;
+ }
+
public DockerRunCommand setOverrideCommandWithArgs(
List<String> overrideCommandWithArgs) {
this.overrrideCommandWithArgs = overrideCommandWithArgs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 560ec18..e8bf564 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1258,6 +1258,7 @@ char* sanitize_docker_command(const char *line) {
{"device", required_argument, 0, 'i' },
{"detach", required_argument, 0, 't' },
{"format", required_argument, 0, 'f' },
+ {"group-add", required_argument, 0, 'x' },
{0, 0, 0, 0}
};
@@ -1357,6 +1358,9 @@ char* sanitize_docker_command(const char *line) {
strcat(output, optarg);
strcat(output, " ");
break;
+ case 'x':
+ quote_and_append_arg(&output, &output_size, "--group-add ", optarg);
+ break;
default:
fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
fflush(LOGFILE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1a72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 0e314bf..e548790 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -77,6 +78,7 @@ public class TestDockerContainerRuntime {
private ContainerLaunchContext context;
private HashMap<String, String> env;
private String image;
+ private String uidGidPair;
private String runAsUser;
private String user;
private String appId;
@@ -126,6 +128,7 @@ public class TestDockerContainerRuntime {
when(context.getEnvironment()).thenReturn(env);
when(container.getUser()).thenReturn(submittingUser);
+ uidGidPair = "";
runAsUser = "run_as_user";
user = "user";
appId = "app_id";
@@ -230,7 +233,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(13, args.size());
//verify arguments
- Assert.assertEquals(runAsUser, args.get(0));
Assert.assertEquals(user, args.get(1));
Assert.assertEquals(Integer.toString(PrivilegedOperation.RunAsUserCommand
.LAUNCH_DOCKER_CONTAINER.getValue()), args.get(2));
@@ -322,6 +324,81 @@ public class TestDockerContainerRuntime {
}
@Test
+ public void testContainerLaunchWithUserRemapping()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ conf.setBoolean(YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING,
+ true);
+ Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
+ new String[]{"whoami"});
+ shexec.execute();
+ // get rid of newline at the end
+ runAsUser = shexec.getOutput().replaceAll("\n$", "");
+ builder.setExecutionAttribute(RUN_AS_USER, runAsUser);
+
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf);
+ runtime.launchContainer(builder.build());
+
+ PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+ List<String> args = op.getArguments();
+ String dockerCommandFile = args.get(11);
+
+ String uid = "";
+ String gid = "";
+ Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-u", runAsUser});
+ Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-g", runAsUser});
+ try {
+ shexec1.execute();
+ // get rid of newline at the end
+ uid = shexec1.getOutput().replaceAll("\n$", "");
+ } catch (Exception e) {
+ LOG.info("Could not run id -u command: " + e);
+ }
+ try {
+ shexec2.execute();
+ // get rid of newline at the end
+ gid = shexec2.getOutput().replaceAll("\n$", "");
+ } catch (Exception e) {
+ LOG.info("Could not run id -g command: " + e);
+ }
+ uidGidPair = uid + ":" + gid;
+
+ //This is the expected docker invocation for this case
+ StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
+ .append("--user=%2$s -d ")
+ .append("--workdir=%3$s ")
+ .append("--net=host ")
+ .append("--hostname=" + defaultHostname + " ")
+ .append(getExpectedTestCapabilitiesArgumentString())
+ .append(getExpectedCGroupsMountString())
+ .append("-v %4$s:%4$s ")
+ .append("-v %5$s:%5$s ")
+ .append("-v %6$s:%6$s ")
+ .append("-v %7$s:%7$s ")
+ .append("-v %8$s:%8$s ")
+ .append("(--group-add \\d+ )*")
+ .append("%9$s ")
+ .append("bash %10$s/launch_container.sh");
+
+ String expectedCommand = String
+ .format(expectedCommandTemplate.toString(), containerId, uidGidPair,
+ containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
+ containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
+ image, containerWorkDir);
+
+ List<String> dockerCommands = Files.readAllLines(
+ Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+
+ Assert.assertEquals(1, dockerCommands.size());
+ //Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+ Assert.assertTrue(dockerCommands.get(0).matches(expectedCommand));
+ }
+
+ @Test
public void testAllowedNetworksConfiguration() throws
ContainerExecutionException {
//the default network configuration should cause
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/13] hadoop git commit: MAPREDUCE-6967. gridmix/SleepReducer
should use Time.monotonicNow for measuring durations. Contributed by Chetna
Chaudhari
Posted by ae...@apache.org.
MAPREDUCE-6967. gridmix/SleepReducer should use Time.monotonicNow for measuring durations. Contributed by Chetna Chaudhari
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b133dc57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b133dc57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b133dc57
Branch: refs/heads/HDFS-7240
Commit: b133dc5700b66318ac6d80e463b049723c511f37
Parents: 08fca50
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Sep 22 15:21:42 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Sep 22 15:21:42 2017 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b133dc57/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
index 6c2a048..87a67f1 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.tools.rumen.TaskAttemptInfo;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Time;
public class SleepJob extends GridmixJob {
public static final Log LOG = LogFactory.getLog(SleepJob.class);
@@ -184,7 +185,7 @@ public class SleepJob extends GridmixJob {
TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS);
//This is to stop accumulating deviation from expected sleep time
//over a period of time.
- long start = System.currentTimeMillis();
+ long start = Time.monotonicNow();
long slept = 0L;
long sleep = 0L;
while (slept < duration) {
@@ -192,7 +193,7 @@ public class SleepJob extends GridmixJob {
sleep = Math.min(rem, RINTERVAL);
context.setStatus("Sleeping... " + rem + " ms left");
TimeUnit.MILLISECONDS.sleep(sleep);
- slept = System.currentTimeMillis() - start;
+ slept = Time.monotonicNow() - start;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/13] hadoop git commit: YARN-7045. Remove
FSLeafQueue#addAppSchedulable. (Contributed by Sen Zhao via Yufei Gu)
Posted by ae...@apache.org.
YARN-7045. Remove FSLeafQueue#addAppSchedulable. (Contributed by Sen Zhao via Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a92ef030
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a92ef030
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a92ef030
Branch: refs/heads/HDFS-7240
Commit: a92ef030a2707182e90acee644e47c8ef7e1fd8d
Parents: 263e2c6
Author: Yufei Gu <yu...@apache.org>
Authored: Thu Sep 21 12:31:07 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Thu Sep 21 12:31:07 2017 -0700
----------------------------------------------------------------------
.../resourcemanager/scheduler/fair/FSLeafQueue.java | 10 ----------
.../resourcemanager/scheduler/fair/TestFSLeafQueue.java | 6 +++---
.../resourcemanager/scheduler/fair/TestFairScheduler.java | 6 +++---
3 files changed, 6 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a92ef030/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 1dcfffc..89ad1e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -94,16 +94,6 @@ public class FSLeafQueue extends FSQueue {
}
}
- // for testing
- void addAppSchedulable(FSAppAttempt appSched) {
- writeLock.lock();
- try {
- runnableApps.add(appSched);
- } finally {
- writeLock.unlock();
- }
- }
-
/**
* Removes the given app from this queue.
* @return whether or not the app was runnable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a92ef030/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 2aed9bf..b6b3f7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -89,8 +89,8 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(maxResource);
- schedulable.addAppSchedulable(app);
- schedulable.addAppSchedulable(app);
+ schedulable.addApp(app, true);
+ schedulable.addApp(app, true);
schedulable.updateDemand();
@@ -165,7 +165,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
@Override
public void run() {
for (int i=0; i < 500; i++) {
- schedulable.addAppSchedulable(app);
+ schedulable.addApp(app, true);
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a92ef030/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 0ef4d7b..d5b1fcc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -5221,12 +5221,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSLeafQueue aQueue =
new FSLeafQueue("root.queue1.a", scheduler, queue1);
aQueue.setMaxShare(maxResource);
- aQueue.addAppSchedulable(app1);
+ aQueue.addApp(app1, true);
FSLeafQueue bQueue =
new FSLeafQueue("root.queue1.b", scheduler, queue1);
bQueue.setMaxShare(maxResource);
- bQueue.addAppSchedulable(app2);
+ bQueue.addApp(app2, true);
queue1.setMaxShare(maxResource);
queue1.addChildQueue(aQueue);
@@ -5270,7 +5270,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(resource);
Mockito.when(app.getResourceUsage()).thenReturn(resource);
- child1.addAppSchedulable(app);
+ child1.addApp(app, true);
child1.updateDemand();
String childQueueString = "{Name: root.parent.child1,"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/13] hadoop git commit: Revert "HADOOP-14655. Update httpcore
version to 4.4.6. (rchiang)"
Posted by ae...@apache.org.
Revert "HADOOP-14655. Update httpcore version to 4.4.6. (rchiang)"
This reverts commit 088a18b44d87c6994b571696edbf87c95528dcaa.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d29bf57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d29bf57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d29bf57
Branch: refs/heads/HDFS-7240
Commit: 8d29bf57ca97a94e6f6ee663bcaa5b7bc390f850
Parents: 4002bf0
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Sep 22 14:43:44 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Sep 22 14:43:44 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d29bf57/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a698126..7f657ca 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -578,7 +578,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.4.6</version>
+ <version>4.4.4</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/13] hadoop git commit: MAPREDUCE-6966. DistSum should use
Time.monotonicNow for measuring durations. Contributed by Chetna Chaudhari.
Posted by ae...@apache.org.
MAPREDUCE-6966. DistSum should use Time.monotonicNow for measuring durations. Contributed by Chetna Chaudhari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c71d1379
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c71d1379
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c71d1379
Branch: refs/heads/HDFS-7240
Commit: c71d13796039815350d3c73c51f9fe48531af0fa
Parents: 53be075
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Sep 22 15:07:59 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Sep 22 15:07:59 2017 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/examples/pi/DistSum.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c71d1379/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
index ffe63fe..9fb6665 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
@@ -162,9 +163,9 @@ public final class DistSum extends Configured implements Tool {
LOG.info(s = "sigma=" + sigma);
context.setStatus(s);
- final long start = System.currentTimeMillis();
+ final long start = Time.monotonicNow();
sigma.compute();
- final long duration = System.currentTimeMillis() - start;
+ final long duration = Time.monotonicNow() - start;
final TaskResult result = new TaskResult(sigma, duration);
LOG.info(s = "result=" + result);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/13] hadoop git commit: MAPREDUCE-6947. Moving logging APIs over to slf4j in hadoop-mapreduce-examples. Contributed by Gergely Novák.
Posted by ae...@apache.org.
MAPREDUCE-6947. Moving logging APIs over to slf4j in hadoop-mapreduce-examples. Contributed by Gergely Novák.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53be0752
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53be0752
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53be0752
Branch: refs/heads/HDFS-7240
Commit: 53be075241f1ba92bfe47e89c2dfc3f0664e2578
Parents: 3baae43
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Sep 22 13:27:59 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Sep 22 13:27:59 2017 +0900
----------------------------------------------------------------------
hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml | 4 ++++
.../java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java | 7 ++++---
.../main/java/org/apache/hadoop/examples/DBCountPageView.java | 7 ++++---
.../java/org/apache/hadoop/examples/dancing/DancingLinks.java | 7 +++----
.../src/main/java/org/apache/hadoop/examples/pi/DistSum.java | 6 +++---
.../java/org/apache/hadoop/examples/terasort/TeraGen.java | 6 +++---
.../org/apache/hadoop/examples/terasort/TeraOutputFormat.java | 7 ++++---
.../org/apache/hadoop/examples/terasort/TeraScheduler.java | 7 ++++---
.../java/org/apache/hadoop/examples/terasort/TeraSort.java | 6 +++---
.../org/apache/hadoop/examples/terasort/TestTeraSort.java | 6 +++---
10 files changed, 35 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index 4a50ed3..5e5dd8f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -112,6 +112,10 @@
<artifactId>guava</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
index b9987a5..2bda89d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
@@ -29,8 +29,6 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -52,6 +50,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Charsets;
@@ -84,7 +84,8 @@ public class BaileyBorweinPlouffe extends Configured implements Tool {
private static final String DIGIT_SIZE_PROPERTY = NAME + ".digit.size";
private static final String DIGIT_PARTS_PROPERTY = NAME + ".digit.parts";
- private static final Log LOG = LogFactory.getLog(BaileyBorweinPlouffe.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BaileyBorweinPlouffe.class);
/** Mapper class computing digits of Pi. */
public static class BbpMapper extends
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java
index 8dec39d..7b73820 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java
@@ -29,8 +29,6 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.LongWritable;
@@ -49,6 +47,8 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.hsqldb.server.Server;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This is a demonstrative program, which uses DBInputFormat for reading
@@ -77,7 +77,8 @@ import org.hsqldb.server.Server;
*/
public class DBCountPageView extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(DBCountPageView.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DBCountPageView.class);
private Connection connection;
private boolean initialized = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DancingLinks.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DancingLinks.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DancingLinks.java
index 0401a25..eef4461 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DancingLinks.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DancingLinks.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.examples.dancing;
import java.util.*;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A generic solver for tile laying problems using Knuth's dancing link
@@ -35,8 +35,7 @@ import org.apache.commons.logging.LogFactory;
* The type parameter ColumnName is the class of application's column names.
*/
public class DancingLinks<ColumnName> {
- private static final Log LOG =
- LogFactory.getLog(DancingLinks.class.getName());
+ private static final Logger LOG = LoggerFactory.getLogger(DancingLinks.class);
/**
* A cell in the table with up/down and left/right links that form doubly
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
index 99f7c24..ffe63fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/DistSum.java
@@ -28,8 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.examples.pi.math.Summation;
@@ -55,6 +53,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The main class for computing sums using map/reduce jobs.
@@ -66,7 +66,7 @@ import org.apache.hadoop.util.ToolRunner;
* a mix-type job may be executed on either side.
*/
public final class DistSum extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(DistSum.class);
+ private static final Logger LOG = LoggerFactory.getLogger(DistSum.class);
private static final String NAME = DistSum.class.getSimpleName();
private static final String N_PARTS = "mapreduce.pi." + NAME + ".nParts";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
index 53bbdc4..ef20c4c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
@@ -25,8 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.zip.Checksum;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -49,6 +47,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.PureJavaCrc32;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Generate the official GraySort input data set.
@@ -66,7 +66,7 @@ import org.apache.hadoop.util.ToolRunner;
* <b>bin/hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir</b>
*/
public class TeraGen extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(TeraGen.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TeraGen.class);
public enum Counters {CHECKSUM}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
index 96580b1..e0ce36c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.examples.terasort;
import java.io.FileNotFoundException;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -38,12 +36,15 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* An output format that writes the key and value appended together.
*/
public class TeraOutputFormat extends FileOutputFormat<Text,Text> {
- private static final Log LOG = LogFactory.getLog(TeraOutputFormat.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TeraOutputFormat.class);
private OutputCommitter committer = null;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
index 3e12a3d..7a9e44b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
@@ -21,17 +21,18 @@ package org.apache.hadoop.examples.terasort;
import java.io.*;
import java.util.*;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Charsets;
class TeraScheduler {
- private static final Log LOG = LogFactory.getLog(TeraScheduler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TeraScheduler.class);
private Split[] splits;
private List<Host> hosts = new ArrayList<Host>();
private int slotsPerHost;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
index 040d13f..8b698e3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
@@ -23,8 +23,6 @@ import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -38,6 +36,8 @@ import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Generates the sampled split points, launches the job, and waits for it to
@@ -47,7 +47,7 @@ import org.apache.hadoop.util.ToolRunner;
* <b>bin/hadoop jar hadoop-*-examples.jar terasort in-dir out-dir</b>
*/
public class TeraSort extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(TeraSort.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TeraSort.class);
/**
* A partitioner that splits text keys into roughly equal partitions
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53be0752/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
index b835a3b..b301659 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.examples.terasort;
import java.io.File;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
@@ -29,12 +27,14 @@ import org.apache.hadoop.mapred.HadoopTestCase;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class TestTeraSort extends HadoopTestCase {
- private static Log LOG = LogFactory.getLog(TestTeraSort.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestTeraSort.class);
public TestTeraSort()
throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/13] hadoop git commit: HDFS-12530. Processor argument in Offline
Image Viewer should be case insensitive. Contributed by Hanisha Koneru.
Posted by ae...@apache.org.
HDFS-12530. Processor argument in Offline Image Viewer should be case insensitive. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08fca508
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08fca508
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08fca508
Branch: refs/heads/HDFS-7240
Commit: 08fca508e66e8eddc5d8fd1608ec0c9cd54fc990
Parents: 908d8e9
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Sep 22 09:47:57 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Sep 22 09:47:57 2017 -0700
----------------------------------------------------------------------
.../tools/offlineImageViewer/OfflineImageViewerPB.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08fca508/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index c1141f3..0f2ac81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.util.StringUtils;
/**
* OfflineImageViewerPB to dump the contents of an Hadoop image file to XML or
@@ -174,8 +175,8 @@ public class OfflineImageViewerPB {
Configuration conf = new Configuration();
try (PrintStream out = outputFile.equals("-") ?
System.out : new PrintStream(outputFile, "UTF-8")) {
- switch (processor) {
- case "FileDistribution":
+ switch (StringUtils.toUpperCase(processor)) {
+ case "FILEDISTRIBUTION":
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
boolean formatOutput = cmd.hasOption("format");
@@ -186,7 +187,7 @@ public class OfflineImageViewerPB {
new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
"r"));
break;
- case "ReverseXML":
+ case "REVERSEXML":
try {
OfflineImageReconstructor.run(inputFile, outputFile);
} catch (Exception e) {
@@ -196,14 +197,14 @@ public class OfflineImageViewerPB {
System.exit(1);
}
break;
- case "Web":
+ case "WEB":
String addr = cmd.getOptionValue("addr", "localhost:5978");
try (WebImageViewer viewer =
new WebImageViewer(NetUtils.createSocketAddr(addr))) {
viewer.start(inputFile);
}
break;
- case "Delimited":
+ case "DELIMITED":
try (PBImageDelimitedTextWriter writer =
new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
writer.visit(new RandomAccessFile(inputFile, "r"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org