You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/09/09 02:37:56 UTC
[01/43] hadoop git commit: HADOOP-13375.
o.a.h.security.TestGroupsCaching.testBackgroundRefreshCounters seems flaky.
(Contributed by Weiwei Yang)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 1dc5c7ebf -> 0f73cd7ff
HADOOP-13375. o.a.h.security.TestGroupsCaching.testBackgroundRefreshCounters seems flaky. (Contributed by Weiwei Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcd21d08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcd21d08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcd21d08
Branch: refs/heads/HDFS-7240
Commit: dcd21d083ab2a66fc3ca3bfda03887461698b7b1
Parents: 08f55cc
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Sep 1 11:03:06 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Sep 1 11:03:06 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/security/Groups.java | 33 +++----
.../hadoop/security/TestGroupsCaching.java | 91 +++++++++++++++++---
2 files changed, 95 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd21d08/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index 4891907..32660c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -43,6 +43,8 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
@@ -355,23 +357,24 @@ public class Groups {
executorService.submit(new Callable<List<String>>() {
@Override
public List<String> call() throws Exception {
- boolean success = false;
- try {
- backgroundRefreshQueued.decrementAndGet();
- backgroundRefreshRunning.incrementAndGet();
- List<String> results = load(key);
- success = true;
- return results;
- } finally {
- backgroundRefreshRunning.decrementAndGet();
- if (success) {
- backgroundRefreshSuccess.incrementAndGet();
- } else {
- backgroundRefreshException.incrementAndGet();
- }
- }
+ backgroundRefreshQueued.decrementAndGet();
+ backgroundRefreshRunning.incrementAndGet();
+ List<String> results = load(key);
+ return results;
}
});
+ Futures.addCallback(listenableFuture, new FutureCallback<List<String>>() {
+ @Override
+ public void onSuccess(List<String> result) {
+ backgroundRefreshSuccess.incrementAndGet();
+ backgroundRefreshRunning.decrementAndGet();
+ }
+ @Override
+ public void onFailure(Throwable t) {
+ backgroundRefreshException.incrementAndGet();
+ backgroundRefreshRunning.decrementAndGet();
+ }
+ });
return listenableFuture;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd21d08/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
index 5a5596e..58c2d1a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
@@ -25,11 +25,16 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeoutException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.FakeTimer;
import org.junit.Before;
import org.junit.Test;
+
+import com.google.common.base.Supplier;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
@@ -67,6 +72,7 @@ public class TestGroupsCaching {
private static int requestCount = 0;
private static long getGroupsDelayMs = 0;
private static boolean throwException;
+ private static volatile CountDownLatch latch = null;
@Override
public List<String> getGroups(String user) throws IOException {
@@ -85,7 +91,19 @@ public class TestGroupsCaching {
return new LinkedList<String>(allGroups);
}
+ /**
+ * Delay returning on a latch or a specific amount of time.
+ */
private void delayIfNecessary() {
+ // cause current method to pause
+ // resume until get notified
+ if (latch != null) {
+ try {
+ latch.await();
+ return;
+ } catch (InterruptedException e) {}
+ }
+
if (getGroupsDelayMs > 0) {
try {
Thread.sleep(getGroupsDelayMs);
@@ -114,6 +132,7 @@ public class TestGroupsCaching {
requestCount = 0;
getGroupsDelayMs = 0;
throwException = false;
+ latch = null;
}
@Override
@@ -142,6 +161,31 @@ public class TestGroupsCaching {
public static void setThrowException(boolean throwIfTrue) {
throwException = throwIfTrue;
}
+
+ /**
+ * Hold on returning the group names unless being notified,
+ * ensure this method is called before {@link #getGroups(String)}.
+ * Call {@link #resume()} will resume the process.
+ */
+ public static void pause() {
+ // Set a static latch, multiple background refresh threads
+ // share this instance. So when await is called, all the
+ // threads will pause until the it decreases the count of
+ // the latch.
+ latch = new CountDownLatch(1);
+ }
+
+ /**
+ * Resume the background refresh thread and return the value
+ * of group names.
+ */
+ public static void resume() {
+ // if latch is null, it means pause was not called and it is
+ // safe to ignore.
+ if (latch != null) {
+ latch.countDown();
+ }
+ }
}
public static class ExceptionalGroupMapping extends ShellBasedUnixGroupsMapping {
@@ -610,22 +654,18 @@ public class TestGroupsCaching {
// expire the cache
timer.advance(2*1000);
- FakeGroupMapping.setGetGroupsDelayMs(40);
+ FakeGroupMapping.pause();
// Request all groups again, as there are 2 threads to process them
// 3 should get queued and 2 should be running
for (String g: grps) {
groups.getGroups(g);
}
- Thread.sleep(20);
- assertEquals(groups.getBackgroundRefreshQueued(), 3);
- assertEquals(groups.getBackgroundRefreshRunning(), 2);
+ waitForGroupCounters(groups, 3, 2, 0, 0);
+ FakeGroupMapping.resume();
- // After 120ms all should have completed running
- Thread.sleep(120);
- assertEquals(groups.getBackgroundRefreshQueued(), 0);
- assertEquals(groups.getBackgroundRefreshRunning(), 0);
- assertEquals(groups.getBackgroundRefreshSuccess(), 5);
+ // Once resumed, all results should be returned immediately
+ waitForGroupCounters(groups, 0, 0, 5, 0);
// Now run again, this time throwing exceptions but no delay
timer.advance(2*1000);
@@ -634,11 +674,34 @@ public class TestGroupsCaching {
for (String g: grps) {
groups.getGroups(g);
}
- Thread.sleep(20);
- assertEquals(groups.getBackgroundRefreshQueued(), 0);
- assertEquals(groups.getBackgroundRefreshRunning(), 0);
- assertEquals(groups.getBackgroundRefreshSuccess(), 5);
- assertEquals(groups.getBackgroundRefreshException(), 5);
+ waitForGroupCounters(groups, 0, 0, 5, 5);
+ }
+
+ private void waitForGroupCounters(Groups groups, long expectedQueued,
+ long expectedRunning, long expectedSuccess, long expectedExpection)
+ throws InterruptedException {
+ long[] expected = {expectedQueued, expectedRunning,
+ expectedSuccess, expectedExpection};
+ long[] actual = new long[expected.length];
+ // wait for a certain time until the counters reach
+ // to expected values. Check values in 20 ms interval.
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ actual[0] = groups.getBackgroundRefreshQueued();
+ actual[1] = groups.getBackgroundRefreshRunning();
+ actual[2] = groups.getBackgroundRefreshSuccess();
+ actual[3] = groups.getBackgroundRefreshException();
+ return Arrays.equals(actual, expected);
+ }
+ }, 20, 1000);
+ } catch (TimeoutException e) {
+ fail("Excepted group counter values are not reached in given time,"
+ + " expecting (Queued, Running, Success, Exception) : "
+ + Arrays.toString(expected) + " but actual : "
+ + Arrays.toString(actual));
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/43] hadoop git commit: YARN-5616. Clean up WeightAdjuster. (Yufei
Gu via kasha)
Posted by ae...@apache.org.
YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0e492e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0e492e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0e492e5
Branch: refs/heads/HDFS-7240
Commit: c0e492e50fa98d423c8a61c94d2d1f9553558b6d
Parents: 680be58
Author: Karthik Kambatla <ka...@apache.org>
Authored: Tue Sep 6 22:40:20 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Tue Sep 6 22:40:20 2016 -0700
----------------------------------------------------------------------
.../scheduler/fair/FairScheduler.java | 5 --
.../scheduler/fair/NewAppWeightBooster.java | 60 --------------------
.../scheduler/fair/WeightAdjuster.java | 36 ------------
3 files changed, 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index ac8dbe1..15e2318 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -186,7 +186,6 @@ public class FairScheduler extends
// an app can be reserved on
protected boolean sizeBasedWeight; // Give larger weights to larger jobs
- protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster
protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not
protected int continuousSchedulingSleepMs; // Sleep time for each pass in continuous scheduling
private Comparator<FSSchedulerNode> nodeAvailableResourceComparator =
@@ -563,10 +562,6 @@ public class FairScheduler extends
weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2);
}
weight *= app.getPriority().getPriority();
- if (weightAdjuster != null) {
- // Run weight through the user-supplied weightAdjuster
- weight = weightAdjuster.adjustWeight(app, weight);
- }
ResourceWeights resourceWeights = app.getResourceWeights();
resourceWeights.setWeight((float)weight);
return resourceWeights;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
deleted file mode 100644
index fb32e56..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-
-/**
- * A {@link WeightAdjuster} implementation that gives a weight boost to new jobs
- * for a certain amount of time -- by default, a 3x weight boost for 60 seconds.
- * This can be used to make shorter jobs finish faster, emulating Shortest Job
- * First scheduling while not starving long jobs.
- */
-@Private
-@Unstable
-public class NewAppWeightBooster extends Configured implements WeightAdjuster {
- private static final float DEFAULT_FACTOR = 3;
- private static final long DEFAULT_DURATION = 5 * 60 * 1000;
-
- private float factor;
- private long duration;
-
- public void setConf(Configuration conf) {
- if (conf != null) {
- factor = conf.getFloat("mapred.newjobweightbooster.factor",
- DEFAULT_FACTOR);
- duration = conf.getLong("mapred.newjobweightbooster.duration",
- DEFAULT_DURATION);
- }
- super.setConf(conf);
- }
-
- public double adjustWeight(FSAppAttempt app, double curWeight) {
- long start = app.getStartTime();
- long now = System.currentTimeMillis();
- if (now - start < duration) {
- return curWeight * factor;
- } else {
- return curWeight;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
deleted file mode 100644
index 67364ed..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configurable;
-
-/**
- * A pluggable object for altering the weights of apps in the fair scheduler,
- * which is used for example by {@link NewAppWeightBooster} to give higher
- * weight to new jobs so that short jobs finish faster.
- *
- * May implement {@link Configurable} to access configuration parameters.
- */
-@Private
-@Unstable
-public interface WeightAdjuster {
- public double adjustWeight(FSAppAttempt app, double curWeight);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/43] hadoop git commit: HADOOP-13447. Refactor S3AFileSystem to
support introduction of separate metadata repository and tests. Contributed
by Chris Nauroth.
Posted by ae...@apache.org.
HADOOP-13447. Refactor S3AFileSystem to support introduction of separate metadata repository and tests. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d152557c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d152557c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d152557c
Branch: refs/heads/HDFS-7240
Commit: d152557cf7f4d2288524c222fcbaf152bdc038b0
Parents: 39d1b1d
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Sep 6 09:36:21 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Sep 6 09:36:21 2016 -0700
----------------------------------------------------------------------
hadoop-tools/hadoop-aws/pom.xml | 5 +
.../org/apache/hadoop/fs/s3a/Constants.java | 29 ++-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 192 ++-------------
.../apache/hadoop/fs/s3a/S3AInputStream.java | 6 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 40 ++++
.../apache/hadoop/fs/s3a/S3ClientFactory.java | 232 +++++++++++++++++++
.../hadoop/fs/s3native/S3xLoginHelper.java | 3 +
.../hadoop/fs/s3a/AbstractS3AMockTest.java | 70 ++++++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 10 +-
.../hadoop/fs/s3a/MockS3ClientFactory.java | 40 ++++
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 126 ++++++++++
12 files changed, 567 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 82a8581..13dcdf1 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -296,6 +296,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 6b8f9f5..64fd8e5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -53,7 +53,8 @@ public final class Constants {
public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15;
// connect to s3 over ssl?
- public static final String SECURE_CONNECTIONS = "fs.s3a.connection.ssl.enabled";
+ public static final String SECURE_CONNECTIONS =
+ "fs.s3a.connection.ssl.enabled";
public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
//use a custom endpoint?
@@ -75,7 +76,8 @@ public final class Constants {
public static final int DEFAULT_MAX_ERROR_RETRIES = 20;
// seconds until we give up trying to establish a connection to s3
- public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout";
+ public static final String ESTABLISH_TIMEOUT =
+ "fs.s3a.connection.establish.timeout";
public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
// seconds until we give up on a connection to s3
@@ -111,11 +113,13 @@ public final class Constants {
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
// minimum size in bytes before we start a multipart uploads or copy
- public static final String MIN_MULTIPART_THRESHOLD = "fs.s3a.multipart.threshold";
+ public static final String MIN_MULTIPART_THRESHOLD =
+ "fs.s3a.multipart.threshold";
public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE;
//enable multiobject-delete calls?
- public static final String ENABLE_MULTI_DELETE = "fs.s3a.multiobjectdelete.enable";
+ public static final String ENABLE_MULTI_DELETE =
+ "fs.s3a.multiobjectdelete.enable";
// comma separated list of directories
public static final String BUFFER_DIR = "fs.s3a.buffer.dir";
@@ -134,11 +138,13 @@ public final class Constants {
public static final String DEFAULT_CANNED_ACL = "";
// should we try to purge old multipart uploads when starting up
- public static final String PURGE_EXISTING_MULTIPART = "fs.s3a.multipart.purge";
+ public static final String PURGE_EXISTING_MULTIPART =
+ "fs.s3a.multipart.purge";
public static final boolean DEFAULT_PURGE_EXISTING_MULTIPART = false;
// purge any multipart uploads older than this number of seconds
- public static final String PURGE_EXISTING_MULTIPART_AGE = "fs.s3a.multipart.purge.age";
+ public static final String PURGE_EXISTING_MULTIPART_AGE =
+ "fs.s3a.multipart.purge.age";
public static final long DEFAULT_PURGE_EXISTING_MULTIPART_AGE = 14400;
// s3 server-side encryption
@@ -198,4 +204,15 @@ public final class Constants {
*/
@InterfaceStability.Unstable
public static final String INPUT_FADV_RANDOM = "random";
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static final String S3_CLIENT_FACTORY_IMPL =
+ "fs.s3a.s3.client.factory.impl";
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static final Class<? extends S3ClientFactory>
+ DEFAULT_S3_CLIENT_FACTORY_IMPL =
+ S3ClientFactory.DefaultS3ClientFactory.class;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
index 5509d36..c25d0fb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
@@ -71,7 +71,7 @@ public class S3AFastOutputStream extends OutputStream {
private static final Logger LOG = S3AFileSystem.LOG;
private final String key;
private final String bucket;
- private final AmazonS3Client client;
+ private final AmazonS3 client;
private final int partSize;
private final int multiPartThreshold;
private final S3AFileSystem fs;
@@ -102,7 +102,7 @@ public class S3AFastOutputStream extends OutputStream {
* @param threadPoolExecutor thread factory
* @throws IOException on any problem
*/
- public S3AFastOutputStream(AmazonS3Client client,
+ public S3AFastOutputStream(AmazonS3 client,
S3AFileSystem fs,
String bucket,
String key,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f8c65da..6a6c26b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -34,11 +34,7 @@ import java.util.concurrent.TimeUnit;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
@@ -57,7 +53,6 @@ import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.event.ProgressEvent;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -78,7 +73,7 @@ import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.ReflectionUtils;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Listing.ACCEPT_ALL;
@@ -110,7 +105,7 @@ public class S3AFileSystem extends FileSystem {
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
private URI uri;
private Path workingDir;
- private AmazonS3Client s3;
+ private AmazonS3 s3;
private String bucket;
private int maxKeys;
private Listing listing;
@@ -147,37 +142,11 @@ public class S3AFileSystem extends FileSystem {
bucket = name.getHost();
- AWSCredentialsProvider credentials =
- createAWSCredentialProviderSet(name, conf, uri);
-
- ClientConfiguration awsConf = new ClientConfiguration();
- awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
- boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
- DEFAULT_SECURE_CONNECTIONS);
- awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
- DEFAULT_MAX_ERROR_RETRIES, 0));
- awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
- DEFAULT_ESTABLISH_TIMEOUT, 0));
- awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
- DEFAULT_SOCKET_TIMEOUT, 0));
- int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
- DEFAULT_SOCKET_SEND_BUFFER, 2048);
- int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
- DEFAULT_SOCKET_RECV_BUFFER, 2048);
- awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
- String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override = {}", signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
-
- initProxySupport(conf, awsConf, secureConnections);
-
- initUserAgent(conf, awsConf);
-
- initAmazonS3Client(conf, credentials, awsConf);
+ Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
+ S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL,
+ S3ClientFactory.class);
+ s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
+ .createS3Client(name, uri);
maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1);
listing = new Listing(this);
@@ -266,50 +235,6 @@ public class S3AFileSystem extends FileSystem {
}
}
- void initProxySupport(Configuration conf, ClientConfiguration awsConf,
- boolean secureConnections) throws IllegalArgumentException {
- String proxyHost = conf.getTrimmed(PROXY_HOST, "");
- int proxyPort = conf.getInt(PROXY_PORT, -1);
- if (!proxyHost.isEmpty()) {
- awsConf.setProxyHost(proxyHost);
- if (proxyPort >= 0) {
- awsConf.setProxyPort(proxyPort);
- } else {
- if (secureConnections) {
- LOG.warn("Proxy host set without port. Using HTTPS default 443");
- awsConf.setProxyPort(443);
- } else {
- LOG.warn("Proxy host set without port. Using HTTP default 80");
- awsConf.setProxyPort(80);
- }
- }
- String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
- String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
- if ((proxyUsername == null) != (proxyPassword == null)) {
- String msg = "Proxy error: " + PROXY_USERNAME + " or " +
- PROXY_PASSWORD + " set without the other.";
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- awsConf.setProxyUsername(proxyUsername);
- awsConf.setProxyPassword(proxyPassword);
- awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
- awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
- "domain {} as workstation {}", awsConf.getProxyHost(),
- awsConf.getProxyPort(),
- String.valueOf(awsConf.getProxyUsername()),
- awsConf.getProxyPassword(), awsConf.getProxyDomain(),
- awsConf.getProxyWorkstation());
- }
- } else if (proxyPort >= 0) {
- String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- }
-
/**
* Get S3A Instrumentation. For test purposes.
* @return this instance's instrumentation.
@@ -318,53 +243,9 @@ public class S3AFileSystem extends FileSystem {
return instrumentation;
}
- /**
- * Initializes the User-Agent header to send in HTTP requests to the S3
- * back-end. We always include the Hadoop version number. The user also may
- * set an optional custom prefix to put in front of the Hadoop version number.
- * The AWS SDK interally appends its own information, which seems to include
- * the AWS SDK version, OS and JVM version.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- */
- private void initUserAgent(Configuration conf, ClientConfiguration awsConf) {
- String userAgent = "Hadoop " + VersionInfo.getVersion();
- String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
- if (!userAgentPrefix.isEmpty()) {
- userAgent = userAgentPrefix + ", " + userAgent;
- }
- LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgent(userAgent);
- }
-
- private void initAmazonS3Client(Configuration conf,
- AWSCredentialsProvider credentials, ClientConfiguration awsConf)
- throws IllegalArgumentException {
- s3 = new AmazonS3Client(credentials, awsConf);
- String endPoint = conf.getTrimmed(ENDPOINT, "");
- if (!endPoint.isEmpty()) {
- try {
- s3.setEndpoint(endPoint);
- } catch (IllegalArgumentException e) {
- String msg = "Incorrect endpoint: " + e.getMessage();
- LOG.error(msg);
- throw new IllegalArgumentException(msg, e);
- }
- }
- enablePathStyleAccessIfRequired(conf);
- }
-
- private void enablePathStyleAccessIfRequired(Configuration conf) {
- final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
- if (pathStyleAccess) {
- LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
- }
- }
-
private void initTransferManager() {
- TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
+ TransferManagerConfiguration transferConfiguration =
+ new TransferManagerConfiguration();
transferConfiguration.setMinimumUploadPartSize(partSize);
transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
transferConfiguration.setMultipartCopyPartSize(partSize);
@@ -435,7 +316,7 @@ public class S3AFileSystem extends FileSystem {
* @return AmazonS3Client
*/
@VisibleForTesting
- AmazonS3Client getAmazonS3Client() {
+ AmazonS3 getAmazonS3Client() {
return s3;
}
@@ -459,10 +340,6 @@ public class S3AFileSystem extends FileSystem {
this.inputPolicy = inputPolicy;
}
- public S3AFileSystem() {
- super();
- }
-
/**
* Turns a path (relative or otherwise) into an S3 key.
*
@@ -782,8 +659,10 @@ public class S3AFileSystem extends FileSystem {
while (true) {
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
- keysToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
- String newDstKey = dstKey + summary.getKey().substring(srcKey.length());
+ keysToDelete.add(
+ new DeleteObjectsRequest.KeyVersion(summary.getKey()));
+ String newDstKey =
+ dstKey + summary.getKey().substring(srcKey.length());
copyFile(summary.getKey(), newDstKey, summary.getSize());
if (keysToDelete.size() == MAX_ENTRIES_TO_DELETE) {
@@ -1388,7 +1267,8 @@ public class S3AFileSystem extends FileSystem {
LOG.debug("Found file (with /): fake directory");
return new S3AFileStatus(true, true, path);
} else {
- LOG.warn("Found file (with /): real file? should not happen: {}", key);
+ LOG.warn("Found file (with /): real file? should not happen: {}",
+ key);
return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
@@ -1985,42 +1865,4 @@ public class S3AFileSystem extends FileSystem {
getFileBlockLocations(status, 0, status.getLen())
: null);
}
-
- /**
- * Get a integer option >= the minimum allowed value.
- * @param conf configuration
- * @param key key to look up
- * @param defVal default value
- * @param min minimum value
- * @return the value
- * @throws IllegalArgumentException if the value is below the minimum
- */
- static int intOption(Configuration conf, String key, int defVal, int min) {
- int v = conf.getInt(key, defVal);
- Preconditions.checkArgument(v >= min,
- String.format("Value of %s: %d is below the minimum value %d",
- key, v, min));
- return v;
- }
-
- /**
- * Get a long option >= the minimum allowed value.
- * @param conf configuration
- * @param key key to look up
- * @param defVal default value
- * @param min minimum value
- * @return the value
- * @throws IllegalArgumentException if the value is below the minimum
- */
- static long longOption(Configuration conf,
- String key,
- long defVal,
- long min) {
- long v = conf.getLong(key, defVal);
- Preconditions.checkArgument(v >= min,
- String.format("Value of %s: %d is below the minimum value %d",
- key, v, min));
- return v;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index ccb9726..dd6cdd7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.google.common.base.Preconditions;
@@ -71,7 +71,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
private volatile boolean closed;
private S3ObjectInputStream wrappedStream;
private final FileSystem.Statistics stats;
- private final AmazonS3Client client;
+ private final AmazonS3 client;
private final String bucket;
private final String key;
private final long contentLength;
@@ -101,7 +101,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
public S3AInputStream(String bucket,
String key,
long contentLength,
- AmazonS3Client client,
+ AmazonS3 client,
FileSystem.Statistics stats,
S3AInstrumentation instrumentation,
long readahead,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 699676d..a5e8e7a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -25,6 +25,9 @@ import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import com.google.common.base.Preconditions;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -403,4 +406,41 @@ public final class S3AUtils {
builder.append("size=").append(summary.getSize());
return builder.toString();
}
+
+ /**
+ * Get a integer option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static int intOption(Configuration conf, String key, int defVal, int min) {
+ int v = conf.getInt(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ return v;
+ }
+
+ /**
+ * Get a long option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static long longOption(Configuration conf,
+ String key,
+ long defVal,
+ long min) {
+ long v = conf.getLong(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ return v;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
new file mode 100644
index 0000000..0a4dd02
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+
+import java.io.IOException;
+import java.net.URI;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.VersionInfo;
+
+import org.slf4j.Logger;
+
+/**
+ * Factory for creation of S3 client instances to be used by {@link S3Store}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+interface S3ClientFactory {
+
+ /**
+ * Creates a new {@link AmazonS3} client. This method accepts the S3A file
+ * system URI both in raw input form and validated form as separate arguments,
+ * because both values may be useful in logging.
+ *
+ * @param name raw input S3A file system URI
+ * @param uri validated form of S3A file system URI
+ * @return S3 client
+ * @throws IOException IO problem
+ */
+ AmazonS3 createS3Client(URI name, URI uri) throws IOException;
+
+ /**
+ * The default factory implementation, which calls the AWS SDK to configure
+ * and create an {@link AmazonS3Client} that communicates with the S3 service.
+ */
+ static class DefaultS3ClientFactory extends Configured
+ implements S3ClientFactory {
+
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ @Override
+ public AmazonS3 createS3Client(URI name, URI uri) throws IOException {
+ Configuration conf = getConf();
+ AWSCredentialsProvider credentials =
+ createAWSCredentialProviderSet(name, conf, uri);
+ ClientConfiguration awsConf = new ClientConfiguration();
+ initConnectionSettings(conf, awsConf);
+ initProxySupport(conf, awsConf);
+ initUserAgent(conf, awsConf);
+ return createAmazonS3Client(conf, credentials, awsConf);
+ }
+
+ /**
+ * Initializes all AWS SDK settings related to connection management.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ */
+ private static void initConnectionSettings(Configuration conf,
+ ClientConfiguration awsConf) {
+ awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
+ DEFAULT_MAXIMUM_CONNECTIONS, 1));
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
+ DEFAULT_SECURE_CONNECTIONS);
+ awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
+ awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
+ DEFAULT_MAX_ERROR_RETRIES, 0));
+ awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
+ DEFAULT_ESTABLISH_TIMEOUT, 0));
+ awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
+ DEFAULT_SOCKET_TIMEOUT, 0));
+ int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
+ DEFAULT_SOCKET_SEND_BUFFER, 2048);
+ int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
+ DEFAULT_SOCKET_RECV_BUFFER, 2048);
+ awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
+ String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
+ if (!signerOverride.isEmpty()) {
+ LOG.debug("Signer override = {}", signerOverride);
+ awsConf.setSignerOverride(signerOverride);
+ }
+ }
+
+ /**
+ * Initializes AWS SDK proxy support if configured.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ * @throws IllegalArgumentException if misconfigured
+ */
+ private static void initProxySupport(Configuration conf,
+ ClientConfiguration awsConf) throws IllegalArgumentException {
+ String proxyHost = conf.getTrimmed(PROXY_HOST, "");
+ int proxyPort = conf.getInt(PROXY_PORT, -1);
+ if (!proxyHost.isEmpty()) {
+ awsConf.setProxyHost(proxyHost);
+ if (proxyPort >= 0) {
+ awsConf.setProxyPort(proxyPort);
+ } else {
+ if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ awsConf.setProxyPort(443);
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ awsConf.setProxyPort(80);
+ }
+ }
+ String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
+ String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME + " or " +
+ PROXY_PASSWORD + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ awsConf.setProxyUsername(proxyUsername);
+ awsConf.setProxyPassword(proxyPassword);
+ awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
+ awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
+ "domain {} as workstation {}", awsConf.getProxyHost(),
+ awsConf.getProxyPort(),
+ String.valueOf(awsConf.getProxyUsername()),
+ awsConf.getProxyPassword(), awsConf.getProxyDomain(),
+ awsConf.getProxyWorkstation());
+ }
+ } else if (proxyPort >= 0) {
+ String msg =
+ "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ /**
+ * Initializes the User-Agent header to send in HTTP requests to the S3
+ * back-end. We always include the Hadoop version number. The user also
+ * may set an optional custom prefix to put in front of the Hadoop version
+ * number. The AWS SDK interally appends its own information, which seems
+ * to include the AWS SDK version, OS and JVM version.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ */
+ private static void initUserAgent(Configuration conf,
+ ClientConfiguration awsConf) {
+ String userAgent = "Hadoop " + VersionInfo.getVersion();
+ String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
+ if (!userAgentPrefix.isEmpty()) {
+ userAgent = userAgentPrefix + ", " + userAgent;
+ }
+ LOG.debug("Using User-Agent: {}", userAgent);
+ awsConf.setUserAgent(userAgent);
+ }
+
+ /**
+ * Creates an {@link AmazonS3Client} from the established configuration.
+ *
+ * @param conf Hadoop configuration
+ * @param credentials AWS credentials
+ * @param awsConf AWS SDK configuration
+ * @return S3 client
+ * @throws IllegalArgumentException if misconfigured
+ */
+ private static AmazonS3 createAmazonS3Client(Configuration conf,
+ AWSCredentialsProvider credentials, ClientConfiguration awsConf)
+ throws IllegalArgumentException {
+ AmazonS3 s3 = new AmazonS3Client(credentials, awsConf);
+ String endPoint = conf.getTrimmed(ENDPOINT, "");
+ if (!endPoint.isEmpty()) {
+ try {
+ s3.setEndpoint(endPoint);
+ } catch (IllegalArgumentException e) {
+ String msg = "Incorrect endpoint: " + e.getMessage();
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg, e);
+ }
+ }
+ enablePathStyleAccessIfRequired(s3, conf);
+ return s3;
+ }
+
+ /**
+ * Enables path-style access to S3 buckets if configured. By default, the
+ * behavior is to use virtual hosted-style access with URIs of the form
+ * http://bucketname.s3.amazonaws.com. Enabling path-style access and a
+ * region-specific endpoint switches the behavior to use URIs of the form
+ * http://s3-eu-west-1.amazonaws.com/bucketname.
+ *
+ * @param s3 S3 client
+ * @param conf Hadoop configuration
+ */
+ private static void enablePathStyleAccessIfRequired(AmazonS3 s3,
+ Configuration conf) {
+ final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
+ if (pathStyleAccess) {
+ LOG.debug("Enabling path style access!");
+ s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index bc8c2e6..97ece37 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -132,6 +132,9 @@ public final class S3xLoginHelper {
*
* This strips out login information.
*
+ * @param uri the URI to canonicalize
+ * @param defaultPort default port to use in canonicalized URI if the input
+ * URI has no port and this value is greater than 0
* @return a new, canonicalized URI.
*/
public static URI canonicalizeUri(URI uri, int defaultPort) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
new file mode 100644
index 0000000..6734947
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AmazonS3;
+
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Abstract base class for S3A unit tests using a mock S3 client.
+ */
+public abstract class AbstractS3AMockTest {
+
+ protected static final String BUCKET = "mock-bucket";
+ protected static final AmazonServiceException NOT_FOUND;
+ static {
+ NOT_FOUND = new AmazonServiceException("Not Found");
+ NOT_FOUND.setStatusCode(404);
+ }
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ protected S3AFileSystem fs;
+ protected AmazonS3 s3;
+
+ @Before
+ public void setup() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
+ S3ClientFactory.class);
+ fs = new S3AFileSystem();
+ URI uri = URI.create(FS_S3A + "://" + BUCKET);
+ fs.initialize(uri, conf);
+ s3 = fs.getAmazonS3Client();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (fs != null) {
+ fs.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 4e99339..fca8e49 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.S3ClientOptions;
import org.apache.commons.lang.StringUtils;
@@ -96,7 +96,7 @@ public class ITestS3AConfiguration {
} else {
conf.set(Constants.ENDPOINT, endpoint);
fs = S3ATestUtils.createTestFileSystem(conf);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
String endPointRegion = "";
// Differentiate handling of "s3-" and "s3." based endpoint identifiers
String[] endpointParts = StringUtils.split(endpoint, '.');
@@ -364,7 +364,7 @@ public class ITestS3AConfiguration {
try {
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
"clientOptions");
@@ -388,7 +388,7 @@ public class ITestS3AConfiguration {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
@@ -401,7 +401,7 @@ public class ITestS3AConfiguration {
conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
new file mode 100644
index 0000000..41f04ee
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.mockito.Mockito.*;
+
+import java.net.URI;
+
+import com.amazonaws.services.s3.AmazonS3;
+
+/**
+ * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3}
+ * interface suitable for unit testing.
+ */
+public class MockS3ClientFactory implements S3ClientFactory {
+
+ @Override
+ public AmazonS3 createS3Client(URI name, URI uri) {
+ String bucket = name.getHost();
+ AmazonS3 s3 = mock(AmazonS3.class);
+ when(s3.doesBucketExist(bucket)).thenReturn(true);
+ return s3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
new file mode 100644
index 0000000..f9e9c6b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+
+import java.io.FileNotFoundException;
+import java.util.Collections;
+import java.util.Date;
+
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.Test;
+
+/**
+ * S3A tests for getFileStatus using mock S3 client.
+ */
+public class TestS3AGetFileStatus extends AbstractS3AMockTest {
+
+ @Test
+ public void testFile() throws Exception {
+ Path path = new Path("/file");
+ String key = path.toUri().getPath().substring(1);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(1L);
+ meta.setLastModified(new Date(2L));
+ when(s3.getObjectMetadata(BUCKET, key)).thenReturn(meta);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isFile());
+ assertEquals(meta.getContentLength(), stat.getLen());
+ assertEquals(meta.getLastModified().getTime(), stat.getModificationTime());
+ }
+
+ @Test
+ public void testFakeDirectory() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(0L);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenReturn(meta);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ }
+
+ @Test
+ public void testImplicitDirectory() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.singletonList("dir/"));
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ }
+
+ @Test
+ public void testRoot() throws Exception {
+ Path path = new Path("/");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.<String>emptyList());
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ assertTrue(stat.getPath().isRoot());
+ }
+
+ @Test
+ public void testNotFound() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.<String>emptyList());
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ exception.expect(FileNotFoundException.class);
+ fs.getFileStatus(path);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/43] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66921274
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66921274
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66921274
Branch: refs/heads/HDFS-7240
Commit: 66921274db47b9a96423e09a802c5af3b9798dd1
Parents: 1dc5c7e f414d5e
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 7 11:19:03 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 7 11:19:03 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 47 +-
.../java/org/apache/hadoop/fs/FileContext.java | 8 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 7 +-
.../hadoop/fs/permission/FsCreateModes.java | 101 +
.../hadoop/fs/permission/FsPermission.java | 16 +-
.../apache/hadoop/fs/permission/RawParser.java | 44 +
.../hadoop/fs/permission/package-info.java | 25 +
.../main/java/org/apache/hadoop/ipc/Client.java | 77 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 246 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 15 +-
.../org/apache/hadoop/ipc/ResponseBuffer.java | 12 +-
.../java/org/apache/hadoop/ipc/RpcWritable.java | 20 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 392 +-
.../java/org/apache/hadoop/security/Groups.java | 33 +-
.../apache/hadoop/security/SaslRpcClient.java | 45 +-
.../hadoop/security/UserGroupInformation.java | 26 +-
.../3.0.0-alpha1/CHANGES.3.0.0-alpha1.md | 3712 ++++++++++++++++++
.../3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md | 1854 +++++++++
.../apache/hadoop/conf/TestConfiguration.java | 46 +
.../fs/TestLocalFileSystemPermission.java | 149 +-
.../hadoop/fs/permission/TestFsPermission.java | 1061 ++---
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 +-
.../hadoop/ipc/TestMultipleProtocolServer.java | 236 +-
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 -
.../apache/hadoop/ipc/TestRPCCompatibility.java | 242 +-
.../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 +-
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 74 +-
.../hadoop/security/TestDoAsEffectiveUser.java | 291 +-
.../hadoop/security/TestGroupsCaching.java | 91 +-
.../security/TestUserGroupInformation.java | 52 +-
.../hadoop-common/src/test/proto/test.proto | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 16 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 6 +-
.../hadoop/hdfs/protocol/DatanodeInfo.java | 33 +-
.../ClientNamenodeProtocolTranslatorPB.java | 14 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 41 +-
.../hdfs/server/protocol/StorageReport.java | 10 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hdfs/web/resources/PermissionParam.java | 9 +-
.../web/resources/UnmaskedPermissionParam.java | 51 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../src/main/proto/hdfs.proto | 2 +
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 14 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
...tNamenodeProtocolServerSideTranslatorPB.java | 16 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 12 +-
.../server/blockmanagement/BlockManager.java | 6 +-
.../BlockPlacementPolicyDefault.java | 5 +-
.../blockmanagement/DatanodeDescriptor.java | 3 +
.../server/blockmanagement/DatanodeManager.java | 9 +-
.../server/blockmanagement/DatanodeStats.java | 6 +-
.../blockmanagement/DatanodeStorageInfo.java | 8 +-
.../blockmanagement/DecommissionManager.java | 7 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 18 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 6 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 24 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 37 +-
.../datanode/web/webhdfs/ParameterParser.java | 7 +
.../datanode/web/webhdfs/WebHdfsHandler.java | 6 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 12 +-
.../hadoop/hdfs/server/namenode/AclStorage.java | 7 +-
.../hdfs/server/namenode/CheckpointConf.java | 10 +-
.../hdfs/server/namenode/FSDirAppendOp.java | 4 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 3 +-
.../hdfs/server/namenode/FSDirSymlinkOp.java | 3 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 5 +-
.../hdfs/server/namenode/FSDirectory.java | 72 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 7 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 -
.../hdfs/server/namenode/ha/EditLogTailer.java | 10 +-
.../web/resources/NamenodeWebHdfsMethods.java | 49 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 +-
.../src/main/resources/hdfs-default.xml | 56 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 6 +
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 10 +-
.../java/org/apache/hadoop/cli/TestAclCLI.java | 6 +-
.../cli/TestAclCLIWithPosixAclInheritance.java | 45 +
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 24 +
.../TestClientProtocolWithDelegationToken.java | 119 -
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../blockmanagement/TestBlockManager.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../extdataset/ExternalDatasetImpl.java | 2 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 49 +-
.../fsdataset/impl/TestFsVolumeList.java | 40 +
.../hdfs/server/namenode/FSAclBaseTest.java | 91 +
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../server/namenode/TestGetBlockLocations.java | 2 +-
.../namenode/TestNamenodeCapacityReport.java | 36 +-
.../namenode/metrics/TestNameNodeMetrics.java | 4 +-
.../snapshot/TestRenameWithSnapshots.java | 4 +-
.../testAclCLIWithPosixAclInheritance.xml | 1075 +++++
.../mapreduce/v2/hs/server/HSAdminServer.java | 3 -
hadoop-project/pom.xml | 7 +
hadoop-tools/hadoop-aws/pom.xml | 9 +
.../org/apache/hadoop/fs/s3a/Constants.java | 29 +-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 192 +-
.../apache/hadoop/fs/s3a/S3AInputStream.java | 6 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 40 +
.../apache/hadoop/fs/s3a/S3ClientFactory.java | 232 ++
.../hadoop/fs/s3native/S3xLoginHelper.java | 3 +
.../hadoop/fs/s3a/AbstractS3AMockTest.java | 70 +
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 10 +-
.../hadoop/fs/s3a/MockS3ClientFactory.java | 40 +
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 126 +
.../hadoop/yarn/conf/YarnConfiguration.java | 12 +
.../yarn/client/api/impl/TestAMRMClient.java | 7 +-
.../src/main/resources/yarn-default.xml | 13 +
.../server/nodemanager/ContainerExecutor.java | 9 +
.../hadoop/yarn/server/nodemanager/Context.java | 2 +
.../nodemanager/DefaultContainerExecutor.java | 6 +
.../nodemanager/DockerContainerExecutor.java | 6 +
.../nodemanager/LinuxContainerExecutor.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 12 +
.../containermanager/ContainerManagerImpl.java | 70 +-
.../containermanager/container/Container.java | 8 +-
.../container/ContainerImpl.java | 207 +-
.../launcher/ContainerLaunch.java | 4 +-
.../localizer/ResourceLocalizationService.java | 21 +-
.../containermanager/localizer/ResourceSet.java | 192 +
.../ContainerLocalizationRequestEvent.java | 2 +-
.../localizer/event/LocalizationEventType.java | 2 +-
.../timelineservice/NMTimelinePublisher.java | 2 +-
.../nodemanager/DummyContainerManager.java | 2 +-
.../TestContainerManagerWithLCE.java | 13 +-
.../amrmproxy/BaseAMRMProxyTest.java | 6 +
.../containermanager/TestContainerManager.java | 137 +-
.../TestResourceLocalizationService.java | 7 +-
.../TestContainersMonitorResourceChange.java | 7 +
.../nodemanager/webapp/MockContainer.java | 17 +-
.../resourcemanager/amlauncher/AMLauncher.java | 26 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 13 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 27 +
.../scheduler/fair/AllocationConfiguration.java | 83 +-
.../scheduler/fair/FSLeafQueue.java | 17 +-
.../scheduler/fair/FSParentQueue.java | 8 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 71 +-
.../scheduler/fair/FairScheduler.java | 12 +-
.../scheduler/fair/MaxRunningAppsEnforcer.java | 11 +-
.../scheduler/fair/NewAppWeightBooster.java | 60 -
.../scheduler/fair/QueueManager.java | 45 +-
.../scheduler/fair/WeightAdjuster.java | 36 -
.../webapp/dao/FairSchedulerQueueInfo.java | 2 +-
.../resourcemanager/TestRMNodeTransitions.java | 29 +-
.../TestResourceTrackerService.java | 4 +-
.../scheduler/capacity/TestLeafQueue.java | 87 +
.../scheduler/fair/TestAppRunnability.java | 6 +-
.../scheduler/fair/TestFSLeafQueue.java | 10 +-
.../scheduler/fair/TestFairScheduler.java | 11 +-
.../fair/TestMaxRunningAppsEnforcer.java | 20 +-
.../scheduler/fair/TestQueueManager.java | 17 +-
157 files changed, 10596 insertions(+), 2778 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66921274/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66921274/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/43] hadoop git commit: Revert "YARN-5567. Fix script exit code
checking in NodeHealthScriptRunner#reportHealthStatus. (Yufei Gu via
rchiang)"
Posted by ae...@apache.org.
Revert "YARN-5567. Fix script exit code checking in NodeHealthScriptRunner#reportHealthStatus. (Yufei Gu via rchiang)"
This reverts commit 05ede003868871addc30162e9707c3dc14ed6b7a.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cae33118
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cae33118
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cae33118
Branch: refs/heads/HDFS-7240
Commit: cae331186da266eea1b0a6fc2c82604907ab0153
Parents: 401db4f
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Sep 8 12:40:30 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 12:40:30 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/util/NodeHealthScriptRunner.java | 3 +--
.../org/apache/hadoop/util/TestNodeHealthScriptRunner.java | 7 -------
2 files changed, 1 insertion(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cae33118/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java
index c3bef37..fc392c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java
@@ -106,7 +106,6 @@ public class NodeHealthScriptRunner extends AbstractService {
shexec.execute();
} catch (ExitCodeException e) {
// ignore the exit code of the script
- exceptionStackTrace = StringUtils.stringifyException(e);
status = HealthCheckerExitStatus.FAILED_WITH_EXIT_CODE;
// On Windows, we will not hit the Stream closed IOException
// thrown by stdout buffered reader for timeout event.
@@ -163,7 +162,7 @@ public class NodeHealthScriptRunner extends AbstractService {
setHealthStatus(false, exceptionStackTrace);
break;
case FAILED_WITH_EXIT_CODE:
- setHealthStatus(false, exceptionStackTrace);
+ setHealthStatus(true, "", now);
break;
case FAILED:
setHealthStatus(false, shexec.getOutput());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cae33118/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNodeHealthScriptRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNodeHealthScriptRunner.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNodeHealthScriptRunner.java
index db61f5a..8fc64d1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNodeHealthScriptRunner.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNodeHealthScriptRunner.java
@@ -91,7 +91,6 @@ public class TestNodeHealthScriptRunner {
public void testNodeHealthScript() throws Exception {
String errorScript = "echo ERROR\n echo \"Tracker not healthy\"";
String normalScript = "echo \"I am all fine\"";
- String failWithExitCodeScript = "echo \"Not healthy\"; exit -1";
String timeOutScript =
Shell.WINDOWS ? "@echo off\nping -n 4 127.0.0.1 >nul\necho \"I am fine\""
: "sleep 4\necho \"I am fine\"";
@@ -125,12 +124,6 @@ public class TestNodeHealthScriptRunner {
nodeHealthScriptRunner.isHealthy());
Assert.assertEquals("", nodeHealthScriptRunner.getHealthReport());
- // Script which fails with exit code.
- writeNodeHealthScriptFile(failWithExitCodeScript, true);
- timerTask.run();
- Assert.assertFalse("Node health status reported healthy",
- nodeHealthScriptRunner.isHealthy());
-
// Timeout script.
writeNodeHealthScriptFile(timeOutScript, true);
timerTask.run();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/43] hadoop git commit: YARN-5555. Scheduler UI: "% of Queue" is
inaccurate if leaf queue is hierarchically nested. Contributed by Eric Payne.
Posted by ae...@apache.org.
YARN-5555. Scheduler UI: "% of Queue" is inaccurate if leaf queue is hierarchically nested. Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05f5c0f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05f5c0f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05f5c0f6
Branch: refs/heads/HDFS-7240
Commit: 05f5c0f631680cffc36a79550c351620615445db
Parents: 0690f09
Author: Varun Vasudev <vv...@apache.org>
Authored: Fri Sep 2 16:02:01 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Fri Sep 2 16:02:01 2016 +0530
----------------------------------------------------------------------
.../scheduler/common/fica/FiCaSchedulerApp.java | 27 ++++++
.../scheduler/capacity/TestLeafQueue.java | 87 ++++++++++++++++++++
2 files changed, 114 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05f5c0f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 33dee80..9c84a23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -57,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAMContainerLaunchDiagnosticsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
@@ -617,4 +619,29 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
updateAMContainerDiagnostics(AMState.ACTIVATED, diagnosticMessageBldr.toString());
}
}
+
+ /**
+ * Recalculates the per-app, percent of queue metric, specific to the
+ * Capacity Scheduler.
+ */
+ @Override
+ public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
+ ApplicationResourceUsageReport report = super.getResourceUsageReport();
+ Resource cluster = rmContext.getScheduler().getClusterResource();
+ Resource totalPartitionRes =
+ rmContext.getNodeLabelManager()
+ .getResourceByLabel(getAppAMNodePartitionName(), cluster);
+ ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
+ if (!calc.isInvalidDivisor(totalPartitionRes)) {
+ float queueAbsMaxCapPerPartition =
+ ((AbstractCSQueue)getQueue()).getQueueCapacities()
+ .getAbsoluteCapacity(getAppAMNodePartitionName());
+ float queueUsagePerc =
+ calc.divide(totalPartitionRes, report.getUsedResources(),
+ Resources.multiply(totalPartitionRes,
+ queueAbsMaxCapPerPartition)) * 100;
+ report.setQueueUsagePercentage(queueUsagePerc);
+ }
+ return report;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05f5c0f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index b2c53da..9134889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -51,6 +51,8 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -67,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -3312,6 +3315,90 @@ public class TestLeafQueue {
return csContext;
}
+ @Test
+ public void testApplicationQueuePercent()
+ throws Exception {
+ Resource res = Resource.newInstance(10 * 1024, 10);
+ CapacityScheduler scheduler = mock(CapacityScheduler.class);
+ when(scheduler.getClusterResource()).thenReturn(res);
+ when(scheduler.getResourceCalculator())
+ .thenReturn(new DefaultResourceCalculator());
+
+ ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3L);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+ when(rmContext.getRMApps())
+ .thenReturn(new ConcurrentHashMap<ApplicationId, RMApp>());
+ RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
+ when(nlm.getResourceByLabel(any(), any())).thenReturn(res);
+ when(rmContext.getNodeLabelManager()).thenReturn(nlm);
+
+ // Queue "test" consumes 100% of the cluster, so its capacity and absolute
+ // capacity are both 1.0f.
+ Queue queue = createQueue("test", null, 1.0f, 1.0f);
+ final String user = "user1";
+ FiCaSchedulerApp app =
+ new FiCaSchedulerApp(appAttId, user, queue,
+ queue.getActiveUsersManager(), rmContext);
+
+ // Resource request
+ Resource requestedResource = Resource.newInstance(1536, 2);
+ app.getAppAttemptResourceUsage().incUsed(requestedResource);
+ // In "test" queue, 1536 used is 15% of both the queue and the cluster
+ assertEquals(15.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+
+ // Queue "test2" is a child of root and its capacity is 50% of root. As a
+ // child of root, its absolute capaicty is also 50%.
+ queue = createQueue("test2", null, 0.5f, 0.5f);
+ app = new FiCaSchedulerApp(appAttId, user, queue,
+ queue.getActiveUsersManager(), rmContext);
+ app.getAppAttemptResourceUsage().incUsed(requestedResource);
+ // In "test2" queue, 1536 used is 30% of "test2" and 15% of the cluster.
+ assertEquals(30.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+
+ // Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
+ // Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
+ AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
+ app = new FiCaSchedulerApp(appAttId, user, qChild,
+ qChild.getActiveUsersManager(), rmContext);
+ app.getAppAttemptResourceUsage().incUsed(requestedResource);
+ // In "test2.1" queue, 1536 used is 60% of "test2.1" and 15% of the cluster.
+ assertEquals(60.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+ }
+
+ private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
+ ApplicationId appIdImpl = ApplicationId.newInstance(0, appId);
+ ApplicationAttemptId attId =
+ ApplicationAttemptId.newInstance(appIdImpl, attemptId);
+ return attId;
+ }
+
+ private AbstractCSQueue createQueue(String name, Queue parent, float capacity,
+ float absCap) {
+ CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
+ QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
+ null, QueueState.RUNNING, null, "", null, false);
+ ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
+ AbstractCSQueue queue = mock(AbstractCSQueue.class);
+ when(queue.getMetrics()).thenReturn(metrics);
+ when(queue.getActiveUsersManager()).thenReturn(activeUsersManager);
+ when(queue.getQueueInfo(false, false)).thenReturn(queueInfo);
+ QueueCapacities qCaps = mock(QueueCapacities.class);
+ when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
+ when(queue.getQueueCapacities()).thenReturn(qCaps);
+ return queue;
+ }
+
@After
public void tearDown() throws Exception {
if (cs != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/43] hadoop git commit: HDFS-10778. Add -format option to make the
output of FileDistribution processor human-readable in OfflineImageViewer.
Posted by ae...@apache.org.
HDFS-10778. Add -format option to make the output of FileDistribution processor human-readable in OfflineImageViewer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63f59489
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63f59489
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63f59489
Branch: refs/heads/HDFS-7240
Commit: 63f594892ecd4687e37a99790288e36eb278849f
Parents: d355573
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Sep 8 15:13:43 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Sep 8 15:13:43 2016 +0900
----------------------------------------------------------------------
.../FileDistributionCalculator.java | 20 +++-
.../FileDistributionVisitor.java | 28 ++++-
.../offlineImageViewer/OfflineImageViewer.java | 116 ++++++++++---------
.../OfflineImageViewerPB.java | 78 +++++++------
.../src/site/markdown/HDFSCommands.md | 1 +
.../src/site/markdown/HdfsImageViewer.md | 1 +
.../TestOfflineImageViewer.java | 24 +++-
7 files changed, 164 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
index 33ab641..71fb822 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
import org.apache.hadoop.util.LimitInputStream;
+import org.apache.hadoop.util.StringUtils;
import com.google.common.base.Preconditions;
@@ -75,11 +76,14 @@ final class FileDistributionCalculator {
private long totalSpace;
private long maxFileSize;
+ private boolean formatOutput = false;
+
FileDistributionCalculator(Configuration conf, long maxSize, int steps,
- PrintStream out) {
+ boolean formatOutput, PrintStream out) {
this.conf = conf;
this.maxSize = maxSize == 0 ? MAX_SIZE_DEFAULT : maxSize;
this.steps = steps == 0 ? INTERVAL_DEFAULT : steps;
+ this.formatOutput = formatOutput;
this.out = out;
long numIntervals = this.maxSize / this.steps;
// avoid OutOfMemoryError when allocating an array
@@ -148,10 +152,20 @@ final class FileDistributionCalculator {
private void output() {
// write the distribution into the output file
- out.print("Size\tNumFiles\n");
+ out.print((formatOutput ? "Size Range" : "Size") + "\tNumFiles\n");
for (int i = 0; i < distribution.length; i++) {
if (distribution[i] != 0) {
- out.print(((long) i * steps) + "\t" + distribution[i]);
+ if (formatOutput) {
+ out.print((i == 0 ? "[" : "(")
+ + StringUtils.byteDesc(((long) (i == 0 ? 0 : i - 1) * steps))
+ + ", "
+ + StringUtils.byteDesc((long)
+ (i == distribution.length - 1 ? maxFileSize : i * steps))
+ + "]\t" + distribution[i]);
+ } else {
+ out.print(((long) i * steps) + "\t" + distribution[i]);
+ }
+
out.print('\n');
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
index 1cef720..7dcc299 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.IOException;
import java.util.LinkedList;
+import org.apache.hadoop.util.StringUtils;
+
/**
* File size distribution visitor.
*
@@ -67,6 +69,7 @@ class FileDistributionVisitor extends TextWriterImageVisitor {
private FileContext current;
private boolean inInode = false;
+ private boolean formatOutput = false;
/**
* File or directory information.
@@ -78,12 +81,12 @@ class FileDistributionVisitor extends TextWriterImageVisitor {
int replication;
}
- public FileDistributionVisitor(String filename,
- long maxSize,
- int step) throws IOException {
+ public FileDistributionVisitor(String filename, long maxSize, int step,
+ boolean formatOutput) throws IOException {
super(filename, false);
this.maxSize = (maxSize == 0 ? MAX_SIZE_DEFAULT : maxSize);
this.step = (step == 0 ? INTERVAL_DEFAULT : step);
+ this.formatOutput = formatOutput;
long numIntervals = this.maxSize / this.step;
if(numIntervals >= Integer.MAX_VALUE)
throw new IOException("Too many distribution intervals " + numIntervals);
@@ -113,9 +116,22 @@ class FileDistributionVisitor extends TextWriterImageVisitor {
private void output() throws IOException {
// write the distribution into the output file
- write("Size\tNumFiles\n");
- for(int i = 0; i < distribution.length; i++)
- write(((long)i * step) + "\t" + distribution[i] + "\n");
+ write((formatOutput ? "Size Range" : "Size") + "\tNumFiles\n");
+ for (int i = 0; i < distribution.length; i++) {
+ if (distribution[i] > 0) {
+ if (formatOutput) {
+ write((i == 0 ? "[" : "(")
+ + StringUtils.byteDesc(((long) (i == 0 ? 0 : i - 1) * step))
+ + ", "
+ + StringUtils.byteDesc((long)
+ (i == distribution.length - 1 ? maxFileSize : i * step))
+ + "]\t"
+ + distribution[i] + "\n");
+ } else {
+ write(((long) i * step) + "\t" + distribution[i] + "\n");
+ }
+ }
+ }
System.out.println("totalFiles = " + totalFiles);
System.out.println("totalDirectories = " + totalDirectories);
System.out.println("totalBlocks = " + totalBlocks);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
index 770cde1..c542d90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
@@ -46,61 +46,63 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.PositionTrackingIn
public class OfflineImageViewer {
public static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
- private final static String usage =
- "Usage: bin/hdfs oiv_legacy [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" +
- "Offline Image Viewer\n" +
- "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n" +
- "saving the results in OUTPUTFILE.\n" +
- "\n" +
- "The oiv utility will attempt to parse correctly formed image files\n" +
- "and will abort fail with mal-formed image files.\n" +
- "\n" +
- "The tool works offline and does not require a running cluster in\n" +
- "order to process an image file.\n" +
- "\n" +
- "The following image processors are available:\n" +
- " * Ls: The default image processor generates an lsr-style listing\n" +
- " of the files in the namespace, with the same fields in the same\n" +
- " order. Note that in order to correctly determine file sizes,\n" +
- " this formatter cannot skip blocks and will override the\n" +
- " -skipBlocks option.\n" +
- " * Indented: This processor enumerates over all of the elements in\n" +
- " the fsimage file, using levels of indentation to delineate\n" +
- " sections within the file.\n" +
- " * Delimited: Generate a text file with all of the elements common\n" +
- " to both inodes and inodes-under-construction, separated by a\n" +
- " delimiter. The default delimiter is \u0001, though this may be\n" +
- " changed via the -delimiter argument. This processor also overrides\n" +
- " the -skipBlocks option for the same reason as the Ls processor\n" +
- " * XML: This processor creates an XML document with all elements of\n" +
- " the fsimage enumerated, suitable for further analysis by XML\n" +
- " tools.\n" +
- " * FileDistribution: This processor analyzes the file size\n" +
- " distribution in the image.\n" +
- " -maxSize specifies the range [0, maxSize] of file sizes to be\n" +
- " analyzed (128GB by default).\n" +
- " -step defines the granularity of the distribution. (2MB by default)\n" +
- " * NameDistribution: This processor analyzes the file names\n" +
- " in the image and prints total number of file names and how frequently\n" +
- " file names are reused.\n" +
- "\n" +
- "Required command line arguments:\n" +
- "-i,--inputFile <arg> FSImage file to process.\n" +
- "-o,--outputFile <arg> Name of output file. If the specified\n" +
- " file exists, it will be overwritten.\n" +
- "\n" +
- "Optional command line arguments:\n" +
- "-p,--processor <arg> Select which type of processor to apply\n" +
- " against image file." +
- " (Ls|XML|Delimited|Indented|FileDistribution).\n" +
- "-h,--help Display usage information and exit\n" +
- "-printToScreen For processors that write to a file, also\n" +
- " output to screen. On large image files this\n" +
- " will dramatically increase processing time.\n" +
- "-skipBlocks Skip inodes' blocks information. May\n" +
- " significantly decrease output.\n" +
- " (default = false).\n" +
- "-delimiter <arg> Delimiting string to use with Delimited processor\n";
+ private final static String usage =
+ "Usage: bin/hdfs oiv_legacy [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n"
+ + "Offline Image Viewer\n"
+ + "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n"
+ + "saving the results in OUTPUTFILE.\n"
+ + "\n"
+ + "The oiv utility will attempt to parse correctly formed image files\n"
+ + "and will abort fail with mal-formed image files.\n"
+ + "\n"
+ + "The tool works offline and does not require a running cluster in\n"
+ + "order to process an image file.\n"
+ + "\n"
+ + "The following image processors are available:\n"
+ + " * Ls: The default image processor generates an lsr-style listing\n"
+ + " of the files in the namespace, with the same fields in the same\n"
+ + " order. Note that in order to correctly determine file sizes,\n"
+ + " this formatter cannot skip blocks and will override the\n"
+ + " -skipBlocks option.\n"
+ + " * Indented: This processor enumerates over all of the elements in\n"
+ + " the fsimage file, using levels of indentation to delineate\n"
+ + " sections within the file.\n"
+ + " * Delimited: Generate a text file with all of the elements common\n"
+ + " to both inodes and inodes-under-construction, separated by a\n"
+ + " delimiter. The default delimiter is \u0001, though this may be\n"
+ + " changed via the -delimiter argument. This processor also overrides\n"
+ + " the -skipBlocks option for the same reason as the Ls processor\n"
+ + " * XML: This processor creates an XML document with all elements of\n"
+ + " the fsimage enumerated, suitable for further analysis by XML\n"
+ + " tools.\n"
+ + " * FileDistribution: This processor analyzes the file size\n"
+ + " distribution in the image.\n"
+ + " -maxSize specifies the range [0, maxSize] of file sizes to be\n"
+ + " analyzed (128GB by default).\n"
+ + " -step defines the granularity of the distribution. (2MB by default)\n"
+ + " -format formats the output result in a human-readable fashion\n"
+ + " rather than a number of bytes. (false by default)\n"
+ + " * NameDistribution: This processor analyzes the file names\n"
+ + " in the image and prints total number of file names and how frequently\n"
+ + " file names are reused.\n"
+ + "\n"
+ + "Required command line arguments:\n"
+ + "-i,--inputFile <arg> FSImage file to process.\n"
+ + "-o,--outputFile <arg> Name of output file. If the specified\n"
+ + " file exists, it will be overwritten.\n"
+ + "\n"
+ + "Optional command line arguments:\n"
+ + "-p,--processor <arg> Select which type of processor to apply\n"
+ + " against image file."
+ + " (Ls|XML|Delimited|Indented|FileDistribution).\n"
+ + "-h,--help Display usage information and exit\n"
+ + "-printToScreen For processors that write to a file, also\n"
+ + " output to screen. On large image files this\n"
+ + " will dramatically increase processing time.\n"
+ + "-skipBlocks Skip inodes' blocks information. May\n"
+ + " significantly decrease output.\n"
+ + " (default = false).\n"
+ + "-delimiter <arg> Delimiting string to use with Delimited processor\n";
private final boolean skipBlocks;
private final String inputFile;
@@ -188,6 +190,7 @@ public class OfflineImageViewer {
options.addOption("h", "help", false, "");
options.addOption("maxSize", true, "");
options.addOption("step", true, "");
+ options.addOption("format", false, "");
options.addOption("skipBlocks", false, "");
options.addOption("printToScreen", false, "");
options.addOption("delimiter", true, "");
@@ -253,7 +256,8 @@ public class OfflineImageViewer {
} else if (processor.equals("FileDistribution")) {
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
- v = new FileDistributionVisitor(outputFile, maxSize, step);
+ boolean formatOutput = cmd.hasOption("format");
+ v = new FileDistributionVisitor(outputFile, maxSize, step, formatOutput);
} else if (processor.equals("NameDistribution")) {
v = new NameDistributionVisitor(outputFile, printToScreen);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index b514b3f..c1141f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -67,6 +67,8 @@ public class OfflineImageViewerPB {
+ " -maxSize specifies the range [0, maxSize] of file sizes to be\n"
+ " analyzed (128GB by default).\n"
+ " -step defines the granularity of the distribution. (2MB by default)\n"
+ + " -format formats the output result in a human-readable fashion\n"
+ + " rather than a number of bytes. (false by default)\n"
+ " * Web: Run a viewer to expose read-only WebHDFS API.\n"
+ " -addr specifies the address to listen. (localhost:5978 by default)\n"
+ " * Delimited (experimental): Generate a text file with all of the elements common\n"
@@ -111,6 +113,7 @@ public class OfflineImageViewerPB {
options.addOption("h", "help", false, "");
options.addOption("maxSize", true, "");
options.addOption("step", true, "");
+ options.addOption("format", false, "");
options.addOption("addr", true, "");
options.addOption("delimiter", true, "");
options.addOption("t", "temp", true, "");
@@ -172,43 +175,44 @@ public class OfflineImageViewerPB {
try (PrintStream out = outputFile.equals("-") ?
System.out : new PrintStream(outputFile, "UTF-8")) {
switch (processor) {
- case "FileDistribution":
- long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
- int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
- new FileDistributionCalculator(conf, maxSize, step, out).visit(
- new RandomAccessFile(inputFile, "r"));
- break;
- case "XML":
- new PBImageXmlWriter(conf, out).visit(
- new RandomAccessFile(inputFile, "r"));
- break;
- case "ReverseXML":
- try {
- OfflineImageReconstructor.run(inputFile, outputFile);
- } catch (Exception e) {
- System.err.println("OfflineImageReconstructor failed: " +
- e.getMessage());
- e.printStackTrace(System.err);
- System.exit(1);
- }
- break;
- case "Web":
- String addr = cmd.getOptionValue("addr", "localhost:5978");
- try (WebImageViewer viewer = new WebImageViewer(
- NetUtils.createSocketAddr(addr))) {
- viewer.start(inputFile);
- }
- break;
- case "Delimited":
- try (PBImageDelimitedTextWriter writer =
- new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
- writer.visit(new RandomAccessFile(inputFile, "r"));
- }
- break;
- default:
- System.err.println("Invalid processor specified : " + processor);
- printUsage();
- return -1;
+ case "FileDistribution":
+ long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
+ int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
+ boolean formatOutput = cmd.hasOption("format");
+ new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
+ .visit(new RandomAccessFile(inputFile, "r"));
+ break;
+ case "XML":
+ new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
+ "r"));
+ break;
+ case "ReverseXML":
+ try {
+ OfflineImageReconstructor.run(inputFile, outputFile);
+ } catch (Exception e) {
+ System.err.println("OfflineImageReconstructor failed: "
+ + e.getMessage());
+ e.printStackTrace(System.err);
+ System.exit(1);
+ }
+ break;
+ case "Web":
+ String addr = cmd.getOptionValue("addr", "localhost:5978");
+ try (WebImageViewer viewer =
+ new WebImageViewer(NetUtils.createSocketAddr(addr))) {
+ viewer.start(inputFile);
+ }
+ break;
+ case "Delimited":
+ try (PBImageDelimitedTextWriter writer =
+ new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
+ writer.visit(new RandomAccessFile(inputFile, "r"));
+ }
+ break;
+ default:
+ System.err.println("Invalid processor specified : " + processor);
+ printUsage();
+ return -1;
}
return 0;
} catch (EOFException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 22886d3..4075878 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -239,6 +239,7 @@ Usage: `hdfs oiv [OPTIONS] -i INPUT_FILE`
| `-addr` *address* | Specify the address(host:port) to listen. (localhost:5978 by default). This option is used with Web processor. |
| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
+| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
| `-delimiter` *arg* | Delimiting string to use with Delimited processor. |
| `-t`,`--temp` *temporary dir* | Use temporary dir to cache intermediate result to generate Delimited outputs. If not set, Delimited processor constructs the namespace in memory before outputting text. |
| `-h`,`--help` | Display the tool usage and help information and exit. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index de27fc2..f991b44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -150,6 +150,7 @@ Options
| `-addr` *address* | Specify the address(host:port) to listen. (localhost:5978 by default). This option is used with Web processor. |
| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
+| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
| `-delimiter` *arg* | Delimiting string to use with Delimited processor. |
| `-t`\|`--temp` *temporary dir* | Use temporary dir to cache intermediate result to generate Delimited outputs. If not set, Delimited processor constructs the namespace in memory before outputting text. |
| `-h`\|`--help` | Display the tool usage and help information and exit. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index a7c30ec..740a8ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -237,7 +237,7 @@ public class TestOfflineImageViewer {
File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile);
- new FileDistributionCalculator(new Configuration(), 0, 0, output)
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
.visit(new RandomAccessFile(truncatedFile, "r"));
}
@@ -259,7 +259,7 @@ public class TestOfflineImageViewer {
public void testFileDistributionCalculator() throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
- new FileDistributionCalculator(new Configuration(), 0, 0, o)
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
.visit(new RandomAccessFile(originalFsimage, "r"));
o.close();
@@ -620,4 +620,24 @@ public class TestOfflineImageViewer {
IOUtils.closeStream(out);
}
}
+
+ @Test
+ public void testOfflineImageViewerWithFormatOption() throws Exception {
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ final PrintStream out = new PrintStream(bytes);
+ final PrintStream oldOut = System.out;
+ try {
+ System.setOut(out);
+ int status =
+ OfflineImageViewerPB.run(new String[] {"-i",
+ originalFsimage.getAbsolutePath(), "-o", "-", "-p",
+ "FileDistribution", "-maxSize", "512", "-step", "8",
+ "-format"});
+ assertEquals(0, status);
+ Assert.assertTrue(bytes.toString().contains("(0 B, 8 B]"));
+ } finally {
+ System.setOut(oldOut);
+ IOUtils.closeStream(out);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/43] hadoop git commit: Revert "HADOOP-13218. Migrate other Hadoop
side tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou
and Kai Zheng"
Posted by ae...@apache.org.
Revert "HADOOP-13218. Migrate other Hadoop side tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and Kai Zheng"
This reverts commit 62a9667136ebd8a048f556b534fcff4fdaf8e2ec
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d355573f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d355573f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d355573f
Branch: refs/heads/HDFS-7240
Commit: d355573f5681f43e760a1bc23ebed553bd35fca5
Parents: f414d5e
Author: Kai Zheng <ka...@intel.com>
Authored: Thu Sep 8 05:50:17 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Thu Sep 8 05:50:17 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 15 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 4 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 ++-
.../hadoop/ipc/TestMultipleProtocolServer.java | 236 ++++++++++++++-
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 +
.../apache/hadoop/ipc/TestRPCCompatibility.java | 242 +++++++++++++--
.../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 +--
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +---
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 74 ++---
.../hadoop/security/TestDoAsEffectiveUser.java | 291 ++++++++++++-------
.../security/TestUserGroupInformation.java | 28 +-
.../hadoop-common/src/test/proto/test.proto | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +
.../TestClientProtocolWithDelegationToken.java | 119 ++++++++
.../mapreduce/v2/hs/server/HSAdminServer.java | 3 +
18 files changed, 876 insertions(+), 294 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index e68bfd4..83e4b9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -60,7 +60,7 @@ public class ProtobufRpcEngine implements RpcEngine {
private static final ThreadLocal<AsyncGet<Message, Exception>>
ASYNC_RETURN_MESSAGE = new ThreadLocal<>();
- static { // Register the rpcRequest deserializer for ProtobufRpcEngine
+ static { // Register the rpcRequest deserializer for WritableRpcEngine
org.apache.hadoop.ipc.Server.registerProtocolEngine(
RPC.RpcKind.RPC_PROTOCOL_BUFFER, RpcProtobufRequest.class,
new Server.ProtoBufRpcInvoker());
@@ -194,8 +194,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (args.length != 2) { // RpcController + Message
- throw new ServiceException(
- "Too many or few parameters for request. Method: ["
+ throw new ServiceException("Too many parameters for request. Method: ["
+ method.getName() + "]" + ", Expected: 2, Actual: "
+ args.length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 12a07a5..3f68d63 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.io.InterruptedIOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
@@ -28,6 +26,7 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.NoRouteToHostException;
import java.net.SocketTimeoutException;
+import java.io.*;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -38,12 +37,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.*;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.*;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolInfoService;
@@ -56,6 +54,7 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
@@ -88,7 +87,7 @@ public class RPC {
RPC_WRITABLE ((short) 2), // Use WritableRpcEngine
RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
- private final short value;
+ public final short value; //TODO make it private
RpcKind(short val) {
this.value = val;
@@ -208,7 +207,7 @@ public class RPC {
RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
if (engine == null) {
Class<?> impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
- ProtobufRpcEngine.class);
+ WritableRpcEngine.class);
engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
PROTOCOL_ENGINES.put(protocol, engine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 531d574..f20ba94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -237,14 +237,14 @@ public abstract class Server {
static class RpcKindMapValue {
final Class<? extends Writable> rpcRequestWrapperClass;
final RpcInvoker rpcInvoker;
-
RpcKindMapValue (Class<? extends Writable> rpcRequestWrapperClass,
RpcInvoker rpcInvoker) {
this.rpcInvoker = rpcInvoker;
this.rpcRequestWrapperClass = rpcRequestWrapperClass;
}
}
- static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new HashMap<>(4);
+ static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new
+ HashMap<RPC.RpcKind, RpcKindMapValue>(4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index ed3a9d0..0ad9abc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -730,7 +730,7 @@ public class UserGroupInformation {
*
* @param user The principal name to load from the ticket
* cache
- * @param ticketCache the path to the ticket cache file
+ * @param ticketCachePath the path to the ticket cache file
*
* @throws IOException if the kerberos login fails
*/
@@ -790,7 +790,7 @@ public class UserGroupInformation {
/**
* Create a UserGroupInformation from a Subject with Kerberos principal.
*
- * @param subject The KerberosPrincipal to use in UGI
+ * @param user The KerberosPrincipal to use in UGI
*
* @throws IOException if the kerberos login fails
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index 9356dab..eb7b949 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -17,8 +17,13 @@
*/
package org.apache.hadoop.ipc;
-import com.google.common.base.Joiner;
-import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -29,6 +34,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoResponseProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
@@ -39,12 +45,8 @@ import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.base.Joiner;
+import com.google.protobuf.BlockingService;
/**
* Benchmark for protobuf RPC.
@@ -66,7 +68,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
public int secondsToRun = 15;
private int msgSize = 1024;
public Class<? extends RpcEngine> rpcEngine =
- ProtobufRpcEngine.class;
+ WritableRpcEngine.class;
private MyOptions(String args[]) {
try {
@@ -133,7 +135,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
opts.addOption(
OptionBuilder.withLongOpt("engine").hasArg(true)
- .withArgName("protobuf")
+ .withArgName("writable|protobuf")
.withDescription("engine to use")
.create('e'));
@@ -182,6 +184,8 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
String eng = line.getOptionValue('e');
if ("protobuf".equals(eng)) {
rpcEngine = ProtobufRpcEngine.class;
+ } else if ("writable".equals(eng)) {
+ rpcEngine = WritableRpcEngine.class;
} else {
throw new ParseException("invalid engine: " + eng);
}
@@ -233,6 +237,11 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
.setInstance(service).setBindAddress(opts.host).setPort(opts.getPort())
.setNumHandlers(opts.serverThreads).setVerbose(false).build();
+ } else if (opts.rpcEngine == WritableRpcEngine.class) {
+ server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestRPC.TestImpl()).setBindAddress(opts.host)
+ .setPort(opts.getPort()).setNumHandlers(opts.serverThreads)
+ .setVerbose(false).build();
} else {
throw new RuntimeException("Bad engine: " + opts.rpcEngine);
}
@@ -390,6 +399,15 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
return responseProto.getMessage();
}
};
+ } else if (opts.rpcEngine == WritableRpcEngine.class) {
+ final TestProtocol proxy = RPC.getProxy(
+ TestProtocol.class, TestProtocol.versionID, addr, conf);
+ return new RpcServiceWrapper() {
+ @Override
+ public String doEcho(String msg) throws Exception {
+ return proxy.echo(msg);
+ }
+ };
} else {
throw new RuntimeException("unsupported engine: " + opts.rpcEngine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
index 10e23ba..8b419e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
@@ -17,28 +17,252 @@
*/
package org.apache.hadoop.ipc;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.junit.Assert;
+
import org.apache.hadoop.conf.Configuration;
-import org.junit.After;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
+import org.apache.hadoop.net.NetUtils;
import org.junit.Before;
+import org.junit.After;
import org.junit.Test;
+import com.google.protobuf.BlockingService;
public class TestMultipleProtocolServer extends TestRpcBase {
-
+ private static InetSocketAddress addr;
private static RPC.Server server;
- @Before
- public void setUp() throws Exception {
- super.setupConf();
+ private static Configuration conf = new Configuration();
+
+
+ @ProtocolInfo(protocolName="Foo")
+ interface Foo0 extends VersionedProtocol {
+ public static final long versionID = 0L;
+ String ping() throws IOException;
+
+ }
+
+ @ProtocolInfo(protocolName="Foo")
+ interface Foo1 extends VersionedProtocol {
+ public static final long versionID = 1L;
+ String ping() throws IOException;
+ String ping2() throws IOException;
+ }
+
+ @ProtocolInfo(protocolName="Foo")
+ interface FooUnimplemented extends VersionedProtocol {
+ public static final long versionID = 2L;
+ String ping() throws IOException;
+ }
+
+ interface Mixin extends VersionedProtocol{
+ public static final long versionID = 0L;
+ void hello() throws IOException;
+ }
+
+ interface Bar extends Mixin {
+ public static final long versionID = 0L;
+ int echo(int i) throws IOException;
+ }
+
+ class Foo0Impl implements Foo0 {
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return Foo0.versionID;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ Class<? extends VersionedProtocol> inter;
+ try {
+ inter = (Class<? extends VersionedProtocol>)getClass().
+ getGenericInterfaces()[0];
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ getProtocolVersion(protocol, clientVersion), inter);
+ }
- server = setupTestServer(conf, 2);
+ @Override
+ public String ping() {
+ return "Foo0";
+ }
+
}
+
+ class Foo1Impl implements Foo1 {
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return Foo1.versionID;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ Class<? extends VersionedProtocol> inter;
+ try {
+ inter = (Class<? extends VersionedProtocol>)getClass().
+ getGenericInterfaces()[0];
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ getProtocolVersion(protocol, clientVersion), inter);
+ }
+
+ @Override
+ public String ping() {
+ return "Foo1";
+ }
+
+ @Override
+ public String ping2() {
+ return "Foo1";
+
+ }
+
+ }
+
+
+ class BarImpl implements Bar {
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return Bar.versionID;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ Class<? extends VersionedProtocol> inter;
+ try {
+ inter = (Class<? extends VersionedProtocol>)getClass().
+ getGenericInterfaces()[0];
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ getProtocolVersion(protocol, clientVersion), inter);
+ }
+
+ @Override
+ public int echo(int i) {
+ return i;
+ }
+
+ @Override
+ public void hello() {
+
+
+ }
+ }
+ @Before
+ public void setUp() throws Exception {
+ // create a server with two handlers
+ server = new RPC.Builder(conf).setProtocol(Foo0.class)
+ .setInstance(new Foo0Impl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Foo1.class, new Foo1Impl());
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Bar.class, new BarImpl());
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Mixin.class, new BarImpl());
+
+
+ // Add Protobuf server
+ // Create server side implementation
+ PBServerImpl pbServerImpl = new PBServerImpl();
+ BlockingService service = TestProtobufRpcProto
+ .newReflectiveBlockingService(pbServerImpl);
+ server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService.class,
+ service);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+ }
+
@After
public void tearDown() throws Exception {
server.stop();
}
+ @Test
+ public void test1() throws IOException {
+ ProtocolProxy<?> proxy;
+ proxy = RPC.getProtocolProxy(Foo0.class, Foo0.versionID, addr, conf);
+ Foo0 foo0 = (Foo0)proxy.getProxy();
+ Assert.assertEquals("Foo0", foo0.ping());
+
+
+ proxy = RPC.getProtocolProxy(Foo1.class, Foo1.versionID, addr, conf);
+
+
+ Foo1 foo1 = (Foo1)proxy.getProxy();
+ Assert.assertEquals("Foo1", foo1.ping());
+ Assert.assertEquals("Foo1", foo1.ping());
+
+
+ proxy = RPC.getProtocolProxy(Bar.class, Foo1.versionID, addr, conf);
+
+
+ Bar bar = (Bar)proxy.getProxy();
+ Assert.assertEquals(99, bar.echo(99));
+
+ // Now test Mixin class method
+
+ Mixin mixin = bar;
+ mixin.hello();
+ }
+
+
+ // Server does not implement the FooUnimplemented version of protocol Foo.
+ // See that calls to it fail.
+ @Test(expected=IOException.class)
+ public void testNonExistingProtocol() throws IOException {
+ ProtocolProxy<?> proxy;
+ proxy = RPC.getProtocolProxy(FooUnimplemented.class,
+ FooUnimplemented.versionID, addr, conf);
+
+ FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
+ foo.ping();
+ }
+
+ /**
+ * getProtocolVersion of an unimplemented version should return highest version
+ * Similarly getProtocolSignature should work.
+ * @throws IOException
+ */
+ @Test
+ public void testNonExistingProtocol2() throws IOException {
+ ProtocolProxy<?> proxy;
+ proxy = RPC.getProtocolProxy(FooUnimplemented.class,
+ FooUnimplemented.versionID, addr, conf);
+
+ FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
+ Assert.assertEquals(Foo1.versionID,
+ foo.getProtocolVersion(RPC.getProtocolName(FooUnimplemented.class),
+ FooUnimplemented.versionID));
+ foo.getProtocolSignature(RPC.getProtocolName(FooUnimplemented.class),
+ FooUnimplemented.versionID, 0);
+ }
+
+ @Test(expected=IOException.class)
+ public void testIncorrectServerCreation() throws IOException {
+ new RPC.Builder(conf).setProtocol(Foo1.class).setInstance(new Foo0Impl())
+ .setBindAddress(ADDRESS).setPort(0).setNumHandlers(2).setVerbose(false)
+ .build();
+ }
+
// Now test a PB service - a server hosts both PB and Writable Rpcs.
@Test
public void testPBService() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
index 6d83d7d..969f728 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
@@ -26,6 +26,19 @@ import org.junit.Test;
public class TestRPCCallBenchmark {
@Test(timeout=20000)
+ public void testBenchmarkWithWritable() throws Exception {
+ int rc = ToolRunner.run(new RPCCallBenchmark(),
+ new String[] {
+ "--clientThreads", "30",
+ "--serverThreads", "30",
+ "--time", "5",
+ "--serverReaderThreads", "4",
+ "--messageSize", "1024",
+ "--engine", "writable"});
+ assertEquals(0, rc);
+ }
+
+ @Test(timeout=20000)
public void testBenchmarkWithProto() throws Exception {
int rc = ToolRunner.run(new RPCCallBenchmark(),
new String[] {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
index a06d9fd..2ac2be9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
@@ -18,20 +18,28 @@
package org.apache.hadoop.ipc;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+
+import org.junit.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
+import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
+import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
+import org.apache.hadoop.net.NetUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
/** Unit test for supporting method-name based compatible RPCs. */
public class TestRPCCompatibility {
private static final String ADDRESS = "0.0.0.0";
@@ -41,7 +49,7 @@ public class TestRPCCompatibility {
public static final Log LOG =
LogFactory.getLog(TestRPCCompatibility.class);
-
+
private static Configuration conf = new Configuration();
public interface TestProtocol0 extends VersionedProtocol {
@@ -112,21 +120,6 @@ public class TestRPCCompatibility {
@Before
public void setUp() {
ProtocolSignature.resetCache();
-
- RPC.setProtocolEngine(conf,
- TestProtocol0.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol1.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol2.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol3.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol4.class, ProtobufRpcEngine.class);
}
@After
@@ -140,7 +133,117 @@ public class TestRPCCompatibility {
server = null;
}
}
+
+ @Test // old client vs new server
+ public void testVersion0ClientVersion1Server() throws Exception {
+ // create a server with two handlers
+ TestImpl1 impl = new TestImpl1();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ proxy = RPC.getProtocolProxy(
+ TestProtocol0.class, TestProtocol0.versionID, addr, conf);
+
+ TestProtocol0 proxy0 = (TestProtocol0)proxy.getProxy();
+ proxy0.ping();
+ }
+
+ @Test // old client vs new server
+ public void testVersion1ClientVersion0Server() throws Exception {
+ // create a server with two handlers
+ server = new RPC.Builder(conf).setProtocol(TestProtocol0.class)
+ .setInstance(new TestImpl0()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ proxy = RPC.getProtocolProxy(
+ TestProtocol1.class, TestProtocol1.versionID, addr, conf);
+
+ TestProtocol1 proxy1 = (TestProtocol1)proxy.getProxy();
+ proxy1.ping();
+ try {
+ proxy1.echo("hello");
+ fail("Echo should fail");
+ } catch(IOException e) {
+ }
+ }
+
+ private class Version2Client {
+ private TestProtocol2 proxy2;
+ private ProtocolProxy<TestProtocol2> serverInfo;
+
+ private Version2Client() throws IOException {
+ serverInfo = RPC.getProtocolProxy(
+ TestProtocol2.class, TestProtocol2.versionID, addr, conf);
+ proxy2 = serverInfo.getProxy();
+ }
+
+ public int echo(int value) throws IOException, NumberFormatException {
+ if (serverInfo.isMethodSupported("echo", int.class)) {
+System.out.println("echo int is supported");
+ return -value; // use version 3 echo long
+ } else { // server is version 2
+System.out.println("echo int is NOT supported");
+ return Integer.parseInt(proxy2.echo(String.valueOf(value)));
+ }
+ }
+
+ public String echo(String value) throws IOException {
+ return proxy2.echo(value);
+ }
+
+ public void ping() throws IOException {
+ proxy2.ping();
+ }
+ }
+
+ @Test // Compatible new client & old server
+ public void testVersion2ClientVersion1Server() throws Exception {
+ // create a server with two handlers
+ TestImpl1 impl = new TestImpl1();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+
+ Version2Client client = new Version2Client();
+ client.ping();
+ assertEquals("hello", client.echo("hello"));
+
+ // echo(int) is not supported by server, so returning 3
+ // This verifies that echo(int) and echo(String)'s hash codes are different
+ assertEquals(3, client.echo(3));
+ }
+
+ @Test // equal version client and server
+ public void testVersion2ClientVersion2Server() throws Exception {
+ // create a server with two handlers
+ TestImpl2 impl = new TestImpl2();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ Version2Client client = new Version2Client();
+
+ client.ping();
+ assertEquals("hello", client.echo("hello"));
+
+ // now that echo(int) is supported by the server, echo(int) should return -3
+ assertEquals(-3, client.echo(3));
+ }
+
public interface TestProtocol3 {
int echo(String value);
int echo(int value);
@@ -194,4 +297,97 @@ public class TestRPCCompatibility {
@Override
int echo(int value) throws IOException;
}
+
+ @Test
+ public void testVersionMismatch() throws IOException {
+ server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
+ .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ TestProtocol4 proxy = RPC.getProxy(TestProtocol4.class,
+ TestProtocol4.versionID, addr, conf);
+ try {
+ proxy.echo(21);
+ fail("The call must throw VersionMismatch exception");
+ } catch (RemoteException ex) {
+ Assert.assertEquals(RPC.VersionMismatch.class.getName(),
+ ex.getClassName());
+ Assert.assertTrue(ex.getErrorCode().equals(
+ RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH));
+ } catch (IOException ex) {
+ fail("Expected version mismatch but got " + ex);
+ }
+ }
+
+ @Test
+ public void testIsMethodSupported() throws IOException {
+ server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
+ .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ TestProtocol2 proxy = RPC.getProxy(TestProtocol2.class,
+ TestProtocol2.versionID, addr, conf);
+ boolean supported = RpcClientUtil.isMethodSupported(proxy,
+ TestProtocol2.class, RPC.RpcKind.RPC_WRITABLE,
+ RPC.getProtocolVersion(TestProtocol2.class), "echo");
+ Assert.assertTrue(supported);
+ supported = RpcClientUtil.isMethodSupported(proxy,
+ TestProtocol2.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ RPC.getProtocolVersion(TestProtocol2.class), "echo");
+ Assert.assertFalse(supported);
+ }
+
+ /**
+ * Verify that ProtocolMetaInfoServerSideTranslatorPB correctly looks up
+ * the server registry to extract protocol signatures and versions.
+ */
+ @Test
+ public void testProtocolMetaInfoSSTranslatorPB() throws Exception {
+ TestImpl1 impl = new TestImpl1();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+
+ ProtocolMetaInfoServerSideTranslatorPB xlator =
+ new ProtocolMetaInfoServerSideTranslatorPB(server);
+
+ GetProtocolSignatureResponseProto resp = xlator.getProtocolSignature(
+ null,
+ createGetProtocolSigRequestProto(TestProtocol1.class,
+ RPC.RpcKind.RPC_PROTOCOL_BUFFER));
+ //No signatures should be found
+ Assert.assertEquals(0, resp.getProtocolSignatureCount());
+ resp = xlator.getProtocolSignature(
+ null,
+ createGetProtocolSigRequestProto(TestProtocol1.class,
+ RPC.RpcKind.RPC_WRITABLE));
+ Assert.assertEquals(1, resp.getProtocolSignatureCount());
+ ProtocolSignatureProto sig = resp.getProtocolSignatureList().get(0);
+ Assert.assertEquals(TestProtocol1.versionID, sig.getVersion());
+ boolean found = false;
+ int expected = ProtocolSignature.getFingerprint(TestProtocol1.class
+ .getMethod("echo", String.class));
+ for (int m : sig.getMethodsList()) {
+ if (expected == m) {
+ found = true;
+ break;
+ }
+ }
+ Assert.assertTrue(found);
+ }
+
+ private GetProtocolSignatureRequestProto createGetProtocolSigRequestProto(
+ Class<?> protocol, RPC.RpcKind rpcKind) {
+ GetProtocolSignatureRequestProto.Builder builder =
+ GetProtocolSignatureRequestProto.newBuilder();
+ builder.setProtocol(protocol.getName());
+ builder.setRpcKind(rpcKind.toString());
+ return builder.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
index b22f91b..5807998 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ipc;
import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -28,13 +30,11 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.nio.channels.ClosedByInterruptException;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
-
/**
* tests that the proxy can be interrupted
*/
-public class TestRPCWaitForProxy extends TestRpcBase {
+public class TestRPCWaitForProxy extends Assert {
+ private static final String ADDRESS = "0.0.0.0";
private static final Logger
LOG = LoggerFactory.getLogger(TestRPCWaitForProxy.class);
@@ -46,15 +46,14 @@ public class TestRPCWaitForProxy extends TestRpcBase {
*
* @throws Throwable any exception other than that which was expected
*/
- @Test(timeout = 50000)
+ @Test(timeout = 10000)
public void testWaitForProxy() throws Throwable {
RpcThread worker = new RpcThread(0);
worker.start();
worker.join();
Throwable caught = worker.getCaught();
- Throwable cause = caught.getCause();
- Assert.assertNotNull("No exception was raised", cause);
- if (!(cause instanceof ConnectException)) {
+ assertNotNull("No exception was raised", caught);
+ if (!(caught instanceof ConnectException)) {
throw caught;
}
}
@@ -70,11 +69,11 @@ public class TestRPCWaitForProxy extends TestRpcBase {
RpcThread worker = new RpcThread(100);
worker.start();
Thread.sleep(1000);
- Assert.assertTrue("worker hasn't started", worker.waitStarted);
+ assertTrue("worker hasn't started", worker.waitStarted);
worker.interrupt();
worker.join();
Throwable caught = worker.getCaught();
- Assert.assertNotNull("No exception was raised", caught);
+ assertNotNull("No exception was raised", caught);
// looking for the root cause here, which can be wrapped
// as part of the NetUtils work. Having this test look
// a the type of exception there would be brittle to improvements
@@ -83,8 +82,6 @@ public class TestRPCWaitForProxy extends TestRpcBase {
if (cause == null) {
// no inner cause, use outer exception as root cause.
cause = caught;
- } else if (cause.getCause() != null) {
- cause = cause.getCause();
}
if (!(cause instanceof InterruptedIOException)
&& !(cause instanceof ClosedByInterruptException)) {
@@ -115,16 +112,12 @@ public class TestRPCWaitForProxy extends TestRpcBase {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
connectRetries);
waitStarted = true;
-
- short invalidPort = 20;
- InetSocketAddress invalidAddress = new InetSocketAddress(ADDRESS,
- invalidPort);
- TestRpcBase.TestRpcService proxy = RPC.getProxy(
- TestRpcBase.TestRpcService.class,
- 1L, invalidAddress, conf);
- // Test echo method
- proxy.echo(null, newEchoRequest("hello"));
-
+ TestProtocol proxy = RPC.waitForProxy(TestProtocol.class,
+ TestProtocol.versionID,
+ new InetSocketAddress(ADDRESS, 20),
+ config,
+ 15000L);
+ proxy.echo("");
} catch (Throwable throwable) {
caught = throwable;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
index 5a8f8d0..bc604a4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -112,8 +112,7 @@ public class TestRpcBase {
return setupTestServer(builder);
}
- protected static RPC.Server setupTestServer(
- RPC.Builder builder) throws IOException {
+ protected static RPC.Server setupTestServer(RPC.Builder builder) throws IOException {
RPC.Server server = builder.build();
server.start();
@@ -176,21 +175,17 @@ public class TestRpcBase {
public TestTokenIdentifier() {
this(new Text(), new Text());
}
-
public TestTokenIdentifier(Text tokenid) {
this(tokenid, new Text());
}
-
public TestTokenIdentifier(Text tokenid, Text realUser) {
this.tokenid = tokenid == null ? new Text() : tokenid;
this.realUser = realUser == null ? new Text() : realUser;
}
-
@Override
public Text getKind() {
return KIND_NAME;
}
-
@Override
public UserGroupInformation getUser() {
if (realUser.toString().isEmpty()) {
@@ -208,7 +203,6 @@ public class TestRpcBase {
tokenid.readFields(in);
realUser.readFields(in);
}
-
@Override
public void write(DataOutput out) throws IOException {
tokenid.write(out);
@@ -240,7 +234,7 @@ public class TestRpcBase {
@SuppressWarnings("unchecked")
@Override
public Token<TestTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
+ Collection<Token<? extends TokenIdentifier>> tokens) {
if (service == null) {
return null;
}
@@ -394,17 +388,19 @@ public class TestRpcBase {
}
@Override
- public TestProtos.UserResponseProto getAuthUser(
+ public TestProtos.AuthUserResponseProto getAuthUser(
RpcController controller, TestProtos.EmptyRequestProto request)
throws ServiceException {
- UserGroupInformation authUser;
+ UserGroupInformation authUser = null;
try {
authUser = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new ServiceException(e);
}
- return newUserResponse(authUser.getUserName());
+ return TestProtos.AuthUserResponseProto.newBuilder()
+ .setAuthUser(authUser.getUserName())
+ .build();
}
@Override
@@ -436,34 +432,6 @@ public class TestRpcBase {
return TestProtos.EmptyResponseProto.newBuilder().build();
}
-
- @Override
- public TestProtos.UserResponseProto getCurrentUser(
- RpcController controller,
- TestProtos.EmptyRequestProto request) throws ServiceException {
- String user;
- try {
- user = UserGroupInformation.getCurrentUser().toString();
- } catch (IOException e) {
- throw new ServiceException("Failed to get current user", e);
- }
-
- return newUserResponse(user);
- }
-
- @Override
- public TestProtos.UserResponseProto getServerRemoteUser(
- RpcController controller,
- TestProtos.EmptyRequestProto request) throws ServiceException {
- String serverRemoteUser = Server.getRemoteUser().toString();
- return newUserResponse(serverRemoteUser);
- }
-
- private TestProtos.UserResponseProto newUserResponse(String user) {
- return TestProtos.UserResponseProto.newBuilder()
- .setUser(user)
- .build();
- }
}
protected static TestProtos.EmptyRequestProto newEmptyRequest() {
@@ -510,4 +478,8 @@ public class TestRpcBase {
}
return null;
}
+
+ protected static String convert(TestProtos.AuthUserResponseProto response) {
+ return response.getAuthUser();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index c48ff2e..72371a7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -45,55 +45,30 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import javax.security.auth.callback.Callback;
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.callback.NameCallback;
-import javax.security.auth.callback.PasswordCallback;
-import javax.security.auth.callback.UnsupportedCallbackException;
-import javax.security.sasl.AuthorizeCallback;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslClient;
-import javax.security.sasl.SaslException;
-import javax.security.sasl.SaslServer;
+import javax.security.auth.callback.*;
+import javax.security.sasl.*;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.security.Security;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import java.util.*;
+import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.KERBEROS;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.SIMPLE;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.TOKEN;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.*;
+import static org.junit.Assert.*;
/** Unit tests for using Sasl over RPC. */
@RunWith(Parameterized.class)
public class TestSaslRPC extends TestRpcBase {
@Parameters
public static Collection<Object[]> data() {
- Collection<Object[]> params = new ArrayList<>();
+ Collection<Object[]> params = new ArrayList<Object[]>();
for (QualityOfProtection qop : QualityOfProtection.values()) {
params.add(new Object[]{ new QualityOfProtection[]{qop},qop, null });
}
@@ -139,7 +114,7 @@ public class TestSaslRPC extends TestRpcBase {
NONE(),
VALID(),
INVALID(),
- OTHER()
+ OTHER();
}
@BeforeClass
@@ -255,7 +230,7 @@ public class TestSaslRPC extends TestRpcBase {
final Server server = setupTestServer(conf, 5, sm);
doDigestRpc(server, sm);
} finally {
- SecurityUtil.setSecurityInfoProviders();
+ SecurityUtil.setSecurityInfoProviders(new SecurityInfo[0]);
}
}
@@ -284,7 +259,7 @@ public class TestSaslRPC extends TestRpcBase {
addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()));
- Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
+ Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
@@ -321,8 +296,8 @@ public class TestSaslRPC extends TestRpcBase {
// set doPing to true
newConf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
- ConnectionId remoteId = ConnectionId.getConnectionId(
- new InetSocketAddress(0), TestRpcService.class, null, 0, null, newConf);
+ ConnectionId remoteId = ConnectionId.getConnectionId(new InetSocketAddress(0),
+ TestRpcService.class, null, 0, null, newConf);
assertEquals(CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT,
remoteId.getPingInterval());
// set doPing to false
@@ -831,13 +806,13 @@ public class TestSaslRPC extends TestRpcBase {
final TestTokenSecretManager sm = new TestTokenSecretManager();
boolean useSecretManager = (serverAuth != SIMPLE);
if (enableSecretManager != null) {
- useSecretManager &= enableSecretManager;
+ useSecretManager &= enableSecretManager.booleanValue();
}
if (forceSecretManager != null) {
- useSecretManager |= forceSecretManager;
+ useSecretManager |= forceSecretManager.booleanValue();
}
final SecretManager<?> serverSm = useSecretManager ? sm : null;
-
+
Server server = serverUgi.doAs(new PrivilegedExceptionAction<Server>() {
@Override
public Server run() throws IOException {
@@ -892,13 +867,13 @@ public class TestSaslRPC extends TestRpcBase {
proxy.ping(null, newEmptyRequest());
// make sure the other side thinks we are who we said we are!!!
assertEquals(clientUgi.getUserName(),
- proxy.getAuthUser(null, newEmptyRequest()).getUser());
+ convert(proxy.getAuthUser(null, newEmptyRequest())));
AuthMethod authMethod =
convert(proxy.getAuthMethod(null, newEmptyRequest()));
// verify sasl completed with correct QOP
assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null,
- RPC.getConnectionIdForProxy(proxy).getSaslQop());
- return authMethod != null ? authMethod.toString() : null;
+ RPC.getConnectionIdForProxy(proxy).getSaslQop());
+ return authMethod.toString();
} catch (ServiceException se) {
if (se.getCause() instanceof RemoteException) {
throw (RemoteException) se.getCause();
@@ -923,18 +898,21 @@ public class TestSaslRPC extends TestRpcBase {
String actual) {
assertEquals(expect.toString(), actual);
}
-
- private static void assertAuthEquals(Pattern expect, String actual) {
+
+ private static void assertAuthEquals(Pattern expect,
+ String actual) {
// this allows us to see the regexp and the value it didn't match
if (!expect.matcher(actual).matches()) {
- fail(); // it failed
+ assertEquals(expect, actual); // it failed
+ } else {
+ assertTrue(true); // it matched
}
}
/*
* Class used to test overriding QOP values using SaslPropertiesResolver
*/
- static class AuthSaslPropertiesResolver extends SaslPropertiesResolver {
+ static class AuthSaslPropertiesResolver extends SaslPropertiesResolver{
@Override
public Map<String, String> getServerProperties(InetAddress address) {
@@ -943,7 +921,7 @@ public class TestSaslRPC extends TestRpcBase {
return newPropertes;
}
}
-
+
public static void main(String[] args) throws Exception {
System.out.println("Testing Kerberos authentication over RPC");
if (args.length != 2) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
index c4dbcac..50d389c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
@@ -17,35 +17,40 @@
*/
package org.apache.hadoop.security;
-import com.google.protobuf.ServiceException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Enumeration;
+
+import org.junit.Assert;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.TestRpcBase;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
-import org.junit.Assert;
+import org.apache.hadoop.security.token.TokenInfo;
import org.junit.Before;
import org.junit.Test;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Enumeration;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenSecretManager;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenSelector;
+import org.apache.commons.logging.*;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/**
- * Test do as effective user.
+ *
*/
-public class TestDoAsEffectiveUser extends TestRpcBase {
+public class TestDoAsEffectiveUser {
final private static String REAL_USER_NAME = "realUser1@HADOOP.APACHE.ORG";
final private static String REAL_USER_SHORT_NAME = "realUser1";
final private static String PROXY_USER_NAME = "proxyUser";
@@ -53,8 +58,8 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
final private static String GROUP2_NAME = "group2";
final private static String[] GROUP_NAMES = new String[] { GROUP1_NAME,
GROUP2_NAME };
-
- private TestRpcService client;
+ private static final String ADDRESS = "0.0.0.0";
+ private TestProtocol proxy;
private static final Configuration masterConf = new Configuration();
@@ -77,7 +82,7 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
private void configureSuperUserIPAddresses(Configuration conf,
String superUserShortName) throws IOException {
- ArrayList<String> ipList = new ArrayList<>();
+ ArrayList<String> ipList = new ArrayList<String>();
Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
.getNetworkInterfaces();
while (netInterfaceList.hasMoreElements()) {
@@ -125,19 +130,50 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
curUGI.toString());
}
- private void checkRemoteUgi(final UserGroupInformation ugi,
- final Configuration conf) throws Exception {
+ @TokenInfo(TestTokenSelector.class)
+ public interface TestProtocol extends VersionedProtocol {
+ public static final long versionID = 1L;
+
+ String aMethod() throws IOException;
+ String getServerRemoteUser() throws IOException;
+ }
+
+ public class TestImpl implements TestProtocol {
+
+ @Override
+ public String aMethod() throws IOException {
+ return UserGroupInformation.getCurrentUser().toString();
+ }
+
+ @Override
+ public String getServerRemoteUser() throws IOException {
+ return Server.getRemoteUser().toString();
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return TestProtocol.versionID;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ return new ProtocolSignature(TestProtocol.versionID, null);
+ }
+ }
+
+ private void checkRemoteUgi(final Server server,
+ final UserGroupInformation ugi, final Configuration conf)
+ throws Exception {
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws ServiceException {
- client = getClient(addr, conf);
- String currentUser = client.getCurrentUser(null,
- newEmptyRequest()).getUser();
- String serverRemoteUser = client.getServerRemoteUser(null,
- newEmptyRequest()).getUser();
-
- Assert.assertEquals(ugi.toString(), currentUser);
- Assert.assertEquals(ugi.toString(), serverRemoteUser);
+ public Void run() throws IOException {
+ proxy = RPC.getProxy(
+ TestProtocol.class, TestProtocol.versionID,
+ NetUtils.getConnectAddress(server), conf);
+ Assert.assertEquals(ugi.toString(), proxy.aMethod());
+ Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser());
return null;
}
});
@@ -149,27 +185,29 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- // Set RPC engine to protobuf RPC engine
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(5).setVerbose(true).build();
refreshConf(conf);
try {
+ server.start();
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(realUserUgi, conf);
+ checkRemoteUgi(server, realUserUgi, conf);
- UserGroupInformation proxyUserUgi =
- UserGroupInformation.createProxyUserForTesting(
+ UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(proxyUserUgi, conf);
+ checkRemoteUgi(server, proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -180,25 +218,29 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(realUserUgi, conf);
+ checkRemoteUgi(server, realUserUgi, conf);
UserGroupInformation proxyUserUgi = UserGroupInformation
.createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(proxyUserUgi, conf);
+ checkRemoteUgi(server, proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -214,14 +256,17 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -230,10 +275,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -241,7 +287,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -250,14 +299,17 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
final Configuration conf = new Configuration();
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 2);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -266,10 +318,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -277,7 +330,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -285,12 +341,15 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
public void testRealUserGroupNotSpecified() throws IOException {
final Configuration conf = new Configuration();
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 2);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -299,10 +358,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = (TestProtocol) RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -310,7 +370,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -321,14 +384,17 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group3");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 2);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -337,10 +403,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -348,7 +415,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -362,17 +432,20 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
final Configuration conf = new Configuration(masterConf);
TestTokenSecretManager sm = new TestTokenSecretManager();
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5, sm);
+ final Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
+
+ server.start();
final UserGroupInformation current = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
-
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<>(tokenId,
+ Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
sm);
SecurityUtil.setTokenService(token, addr);
UserGroupInformation proxyUserUgi = UserGroupInformation
@@ -380,19 +453,23 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
proxyUserUgi.addToken(token);
refreshConf(conf);
-
+
String retVal = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
});
@@ -409,34 +486,42 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
TestTokenSecretManager sm = new TestTokenSecretManager();
final Configuration newConf = new Configuration(masterConf);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, newConf);
- // Set RPC engine to protobuf RPC engine
- RPC.setProtocolEngine(newConf, TestRpcService.class,
- ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(newConf);
- final Server server = setupTestServer(newConf, 5, sm);
+ final Server server = new RPC.Builder(newConf)
+ .setProtocol(TestProtocol.class).setInstance(new TestImpl())
+ .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
+ .setSecretManager(sm).build();
+
+ server.start();
final UserGroupInformation current = UserGroupInformation
.createUserForTesting(REAL_USER_NAME, GROUP_NAMES);
refreshConf(newConf);
-
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
+ Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
+ sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
String retVal = current.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- client = getClient(addr, newConf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, newConf);
+ String ret = proxy.aMethod();
+ return ret;
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index e45d70d..b3ea5f2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -20,7 +20,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -29,11 +28,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -55,22 +50,9 @@ import java.util.Set;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
-import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
-import static org.apache.hadoop.test.MetricsAsserts.assertGaugeGt;
-import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
-import static org.apache.hadoop.test.MetricsAsserts.getDoubleGauge;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.apache.hadoop.ipc.TestSaslRPC.*;
+import static org.apache.hadoop.test.MetricsAsserts.*;
+import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -127,7 +109,7 @@ public class TestUserGroupInformation {
UserGroupInformation.setLoginUser(null);
}
- @Test(timeout = 30000)
+ @Test (timeout = 30000)
public void testSimpleLogin() throws IOException {
tryLoginAuthenticationMethod(AuthenticationMethod.SIMPLE, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/proto/test.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test.proto b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
index 6411f97..99cd93d 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
@@ -88,6 +88,6 @@ message AuthMethodResponseProto {
required string mechanismName = 2;
}
-message UserResponseProto {
- required string user = 1;
+message AuthUserResponseProto {
+ required string authUser = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 06f6c4f..3292115 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -40,11 +40,9 @@ service TestProtobufRpcProto {
rpc exchange(ExchangeRequestProto) returns (ExchangeResponseProto);
rpc sleep(SleepRequestProto) returns (EmptyResponseProto);
rpc getAuthMethod(EmptyRequestProto) returns (AuthMethodResponseProto);
- rpc getAuthUser(EmptyRequestProto) returns (UserResponseProto);
+ rpc getAuthUser(EmptyRequestProto) returns (AuthUserResponseProto);
rpc echoPostponed(EchoRequestProto) returns (EchoResponseProto);
rpc sendPostponed(EmptyRequestProto) returns (EmptyResponseProto);
- rpc getCurrentUser(EmptyRequestProto) returns (UserResponseProto);
- rpc getServerRemoteUser(EmptyRequestProto) returns (UserResponseProto);
}
service TestProtobufRpc2Proto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 57f7cb1..6b52949 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -168,6 +168,7 @@ import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
import org.apache.hadoop.net.Node;
@@ -316,6 +317,8 @@ public class NameNodeRpcServer implements NamenodeProtocols {
new TraceAdminProtocolServerSideTranslatorPB(this);
BlockingService traceAdminService = TraceAdminService
.newReflectiveBlockingService(traceAdminXlator);
+
+ WritableRpcEngine.ensureInitialized();
InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
if (serviceRpcAddr != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
new file mode 100644
index 0000000..0b7ee33
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.mockito.Mockito.mock;
+
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SaslInputStream;
+import org.apache.hadoop.security.SaslRpcClient;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.Test;
+
+/** Unit tests for using Delegation Token over RPC. */
+public class TestClientProtocolWithDelegationToken {
+ private static final String ADDRESS = "0.0.0.0";
+
+ public static final Log LOG = LogFactory
+ .getLog(TestClientProtocolWithDelegationToken.class);
+
+ private static final Configuration conf;
+ static {
+ conf = new Configuration();
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ static {
+ GenericTestUtils.setLogLevel(Client.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(Server.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(SaslRpcClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(SaslRpcServer.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(SaslInputStream.LOG, Level.ALL);
+ }
+
+ @Test
+ public void testDelegationTokenRpc() throws Exception {
+ ClientProtocol mockNN = mock(ClientProtocol.class);
+ FSNamesystem mockNameSys = mock(FSNamesystem.class);
+
+ DelegationTokenSecretManager sm = new DelegationTokenSecretManager(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
+ 3600000, mockNameSys);
+ sm.startThreads();
+ final Server server = new RPC.Builder(conf)
+ .setProtocol(ClientProtocol.class).setInstance(mockNN)
+ .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
+ .setSecretManager(sm).build();
+
+ server.start();
+
+ final UserGroupInformation current = UserGroupInformation.getCurrentUser();
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ String user = current.getUserName();
+ Text owner = new Text(user);
+ DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null);
+ Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
+ dtId, sm);
+ SecurityUtil.setTokenService(token, addr);
+ LOG.info("Service for token is " + token.getService());
+ current.addToken(token);
+ current.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ ClientProtocol proxy = null;
+ try {
+ proxy = RPC.getProxy(ClientProtocol.class,
+ ClientProtocol.versionID, addr, conf);
+ proxy.getServerDefaults();
+ } finally {
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
+ }
+ return null;
+ }
+ });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
index 729af0a..3fef5e2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
@@ -97,6 +98,8 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
BlockingService refreshHSAdminProtocolService = HSAdminRefreshProtocolService
.newReflectiveBlockingService(refreshHSAdminProtocolXlator);
+ WritableRpcEngine.ensureInitialized();
+
clientRpcAddress = conf.getSocketAddr(
JHAdminConfig.MR_HISTORY_BIND_HOST,
JHAdminConfig.JHS_ADMIN_ADDRESS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/43] hadoop git commit: YARN-5608. TestAMRMClient.setup() fails
with ArrayOutOfBoundsException. Contributed by Daniel Templeton.
Posted by ae...@apache.org.
YARN-5608. TestAMRMClient.setup() fails with ArrayOutOfBoundsException. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9557127
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9557127
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9557127
Branch: refs/heads/HDFS-7240
Commit: f9557127b3660e138cab8d9535a589677809384a
Parents: 07650bc
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Sep 6 14:09:45 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Sep 6 14:09:45 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TestAMRMClient.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9557127/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 38178a4..239189a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -145,8 +145,13 @@ public class TestAMRMClient {
yarnClient.start();
// get node info
+ assertTrue("All node managers did not connect to the RM within the "
+ + "allotted 5-second timeout",
+ yarnCluster.waitForNodeManagersToConnect(5000L));
nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
-
+ assertEquals("Not all node managers were reported running",
+ nodeCount, nodeReports.size());
+
priority = Priority.newInstance(1);
priority2 = Priority.newInstance(2);
capability = Resource.newInstance(1024, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/43] hadoop git commit: HDFS-9038. DFS reserved space is
erroneously counted towards non-DFS used. (Brahma Reddy Battula)
Posted by ae...@apache.org.
HDFS-9038. DFS reserved space is erroneously counted towards non-DFS used. (Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f23abfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f23abfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f23abfa
Branch: refs/heads/HDFS-7240
Commit: 5f23abfa30ea29a5474513c463b4d462c0e824ee
Parents: f6c0b75
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 6 13:37:21 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 6 13:37:21 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 33 +++++++++++++---
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 41 +++++++++++++-------
.../hdfs/server/protocol/StorageReport.java | 10 ++++-
.../src/main/proto/hdfs.proto | 2 +
.../blockmanagement/DatanodeDescriptor.java | 3 ++
.../server/blockmanagement/DatanodeStats.java | 6 ++-
.../blockmanagement/DatanodeStorageInfo.java | 8 +++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 3 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 37 ++++++++++++++++--
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 24 ++++++++++++
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../blockmanagement/TestBlockManager.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../extdataset/ExternalDatasetImpl.java | 2 +-
.../fsdataset/impl/TestFsVolumeList.java | 40 +++++++++++++++++++
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../namenode/TestNamenodeCapacityReport.java | 36 ++++++++++++++---
.../namenode/metrics/TestNameNodeMetrics.java | 4 +-
19 files changed, 219 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index cd32a53..e9ee8b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -43,6 +43,7 @@ import static org.apache.hadoop.hdfs.DFSUtilClient.percent2String;
public class DatanodeInfo extends DatanodeID implements Node {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private long remaining;
private long blockPoolUsed;
private long cacheCapacity;
@@ -89,6 +90,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(from);
this.capacity = from.getCapacity();
this.dfsUsed = from.getDfsUsed();
+ this.nonDfsUsed = from.getNonDfsUsed();
this.remaining = from.getRemaining();
this.blockPoolUsed = from.getBlockPoolUsed();
this.cacheCapacity = from.getCacheCapacity();
@@ -105,6 +107,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(nodeID);
this.capacity = 0L;
this.dfsUsed = 0L;
+ this.nonDfsUsed = 0L;
this.remaining = 0L;
this.blockPoolUsed = 0L;
this.cacheCapacity = 0L;
@@ -158,10 +161,26 @@ public class DatanodeInfo extends DatanodeID implements Node {
final int xceiverCount, final String networkLocation,
final AdminStates adminState,
final String upgradeDomain) {
- super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
- infoSecurePort, ipcPort);
+ this(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort, capacity, dfsUsed, 0L, remaining, blockPoolUsed,
+ cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic,
+ xceiverCount, networkLocation, adminState, upgradeDomain);
+ }
+
+ /** Constructor. */
+ public DatanodeInfo(final String ipAddr, final String hostName,
+ final String datanodeUuid, final int xferPort, final int infoPort,
+ final int infoSecurePort, final int ipcPort, final long capacity,
+ final long dfsUsed, final long nonDfsUsed, final long remaining,
+ final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+ final long lastUpdate, final long lastUpdateMonotonic,
+ final int xceiverCount, final String networkLocation,
+ final AdminStates adminState, final String upgradeDomain) {
+ super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort);
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = blockPoolUsed;
this.cacheCapacity = cacheCapacity;
@@ -174,7 +193,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.upgradeDomain = upgradeDomain;
}
- /** Network location name */
+ /** Network location name. */
@Override
public String getName() {
return getXferAddr();
@@ -191,8 +210,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
/** The used space by the data node. */
public long getNonDfsUsed() {
- long nonDFSUsed = capacity - dfsUsed - remaining;
- return nonDFSUsed < 0 ? 0 : nonDFSUsed;
+ return nonDfsUsed;
}
/** The used space by the data node as percentage of present capacity */
@@ -282,6 +300,11 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.dfsUsed = dfsUsed;
}
+ /** Sets the nondfs-used space for the datanode. */
+ public void setNonDfsUsed(long nonDfsUsed) {
+ this.nonDfsUsed = nonDfsUsed;
+ }
+
/** Sets raw free space. */
public void setRemaining(long remaining) {
this.remaining = remaining;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 7df68e7..246b5a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -289,6 +289,7 @@ public class PBHelperClient {
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
.setDfsUsed(info.getDfsUsed())
+ .setNonDfsUsed(info.getNonDfsUsed())
.setRemaining(info.getRemaining())
.setBlockPoolUsed(info.getBlockPoolUsed())
.setCacheCapacity(info.getCacheCapacity())
@@ -581,15 +582,24 @@ public class PBHelperClient {
}
static public DatanodeInfo convert(DatanodeInfoProto di) {
- if (di == null) return null;
- return new DatanodeInfo(
- convert(di.getId()),
- di.hasLocation() ? di.getLocation() : null,
- di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
- di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
- di.getLastUpdate(), di.getLastUpdateMonotonic(),
- di.getXceiverCount(), convert(di.getAdminState()),
+ if (di == null) {
+ return null;
+ }
+ DatanodeInfo dinfo = new DatanodeInfo(convert(di.getId()),
+ di.hasLocation() ? di.getLocation() : null, di.getCapacity(),
+ di.getDfsUsed(), di.getRemaining(), di.getBlockPoolUsed(),
+ di.getCacheCapacity(), di.getCacheUsed(), di.getLastUpdate(),
+ di.getLastUpdateMonotonic(), di.getXceiverCount(),
+ convert(di.getAdminState()),
di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
+ if (di.hasNonDfsUsed()) {
+ dinfo.setNonDfsUsed(di.getNonDfsUsed());
+ } else {
+ // use the legacy way for older datanodes
+ long nonDFSUsed = di.getCapacity() - di.getDfsUsed() - di.getRemaining();
+ dinfo.setNonDfsUsed(nonDFSUsed < 0 ? 0 : nonDFSUsed);
+ }
+ return dinfo;
}
public static StorageType[] convertStorageTypes(
@@ -1556,12 +1566,12 @@ public class PBHelperClient {
}
public static StorageReport convert(StorageReportProto p) {
- return new StorageReport(
- p.hasStorage() ?
- convert(p.getStorage()) :
- new DatanodeStorage(p.getStorageUuid()),
- p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
- p.getBlockPoolUsed());
+ long nonDfsUsed = p.hasNonDfsUsed() ? p.getNonDfsUsed() : p.getCapacity()
+ - p.getDfsUsed() - p.getRemaining();
+ return new StorageReport(p.hasStorage() ? convert(p.getStorage())
+ : new DatanodeStorage(p.getStorageUuid()), p.getFailed(),
+ p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
+ p.getBlockPoolUsed(), nonDfsUsed);
}
public static DatanodeStorage convert(DatanodeStorageProto s) {
@@ -2136,7 +2146,8 @@ public class PBHelperClient {
.setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
.setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
.setStorageUuid(r.getStorage().getStorageID())
- .setStorage(convert(r.getStorage()));
+ .setStorage(convert(r.getStorage()))
+ .setNonDfsUsed(r.getNonDfsUsed());
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
index 3fbc424..7042b58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
@@ -25,17 +25,19 @@ public class StorageReport {
private final boolean failed;
private final long capacity;
private final long dfsUsed;
+ private final long nonDfsUsed;
private final long remaining;
private final long blockPoolUsed;
public static final StorageReport[] EMPTY_ARRAY = {};
- public StorageReport(DatanodeStorage storage, boolean failed,
- long capacity, long dfsUsed, long remaining, long bpUsed) {
+ public StorageReport(DatanodeStorage storage, boolean failed, long capacity,
+ long dfsUsed, long remaining, long bpUsed, long nonDfsUsed) {
this.storage = storage;
this.failed = failed;
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = bpUsed;
}
@@ -56,6 +58,10 @@ public class StorageReport {
return dfsUsed;
}
+ public long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
public long getRemaining() {
return remaining;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 7de8550..1141fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -101,6 +101,7 @@ message DatanodeInfoProto {
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
optional string upgradeDomain = 14;
+ optional uint64 nonDfsUsed = 15;
}
/**
@@ -125,6 +126,7 @@ message StorageReportProto {
optional uint64 remaining = 5 [ default = 0 ];
optional uint64 blockPoolUsed = 6 [ default = 0 ];
optional DatanodeStorageProto storage = 7; // supersedes StorageUuid
+ optional uint64 nonDfsUsed = 8;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 1646129..c74d7c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -417,6 +417,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
long totalRemaining = 0;
long totalBlockPoolUsed = 0;
long totalDfsUsed = 0;
+ long totalNonDfsUsed = 0;
Set<DatanodeStorageInfo> failedStorageInfos = null;
// Decide if we should check for any missing StorageReport and mark it as
@@ -477,6 +478,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
totalRemaining += report.getRemaining();
totalBlockPoolUsed += report.getBlockPoolUsed();
totalDfsUsed += report.getDfsUsed();
+ totalNonDfsUsed += report.getNonDfsUsed();
}
rollBlocksScheduled(getLastUpdateMonotonic());
@@ -485,6 +487,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
setRemaining(totalRemaining);
setBlockPoolUsed(totalBlockPoolUsed);
setDfsUsed(totalDfsUsed);
+ setNonDfsUsed(totalNonDfsUsed);
if (checkFailedStorages) {
updateFailedStorage(failedStorageInfos);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
index 0d4e235..8386b27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
@@ -35,6 +35,7 @@ class DatanodeStats {
private final StorageTypeStatsMap statsMap = new StorageTypeStatsMap();
private long capacityTotal = 0L;
private long capacityUsed = 0L;
+ private long capacityUsedNonDfs = 0L;
private long capacityRemaining = 0L;
private long blockPoolUsed = 0L;
private int xceiverCount = 0;
@@ -49,6 +50,7 @@ class DatanodeStats {
xceiverCount += node.getXceiverCount();
if (node.isInService()) {
capacityUsed += node.getDfsUsed();
+ capacityUsedNonDfs += node.getNonDfsUsed();
blockPoolUsed += node.getBlockPoolUsed();
nodesInService++;
nodesInServiceXceiverCount += node.getXceiverCount();
@@ -77,6 +79,7 @@ class DatanodeStats {
xceiverCount -= node.getXceiverCount();
if (node.isInService()) {
capacityUsed -= node.getDfsUsed();
+ capacityUsedNonDfs -= node.getNonDfsUsed();
blockPoolUsed -= node.getBlockPoolUsed();
nodesInService--;
nodesInServiceXceiverCount -= node.getXceiverCount();
@@ -159,8 +162,7 @@ class DatanodeStats {
}
synchronized long getCapacityUsedNonDFS() {
- final long nonDFSUsed = capacityTotal - capacityRemaining - capacityUsed;
- return nonDFSUsed < 0L? 0L : nonDFSUsed;
+ return capacityUsedNonDfs;
}
synchronized float getCapacityUsedPercent() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 843a8d5..d98a2c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -93,6 +93,7 @@ public class DatanodeStorageInfo {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private volatile long remaining;
private long blockPoolUsed;
@@ -202,6 +203,10 @@ public class DatanodeStorageInfo {
return dfsUsed;
}
+ long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
long getRemaining() {
return remaining;
}
@@ -283,6 +288,7 @@ public class DatanodeStorageInfo {
void updateState(StorageReport r) {
capacity = r.getCapacity();
dfsUsed = r.getDfsUsed();
+ nonDfsUsed = r.getNonDfsUsed();
remaining = r.getRemaining();
blockPoolUsed = r.getBlockPoolUsed();
}
@@ -332,7 +338,7 @@ public class DatanodeStorageInfo {
StorageReport toStorageReport() {
return new StorageReport(
new DatanodeStorage(storageID, state, storageType),
- false, capacity, dfsUsed, remaining, blockPoolUsed);
+ false, capacity, dfsUsed, remaining, blockPoolUsed, nonDfsUsed);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index e0d2baf..ffd2f8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -164,7 +164,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volume.getCapacity(),
volume.getDfsUsed(),
volume.getAvailable(),
- volume.getBlockPoolUsed(bpid));
+ volume.getBlockPoolUsed(bpid),
+ volume.getNonDfsUsed());
reports.add(sr);
} catch (ClosedChannelException e) {
continue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 59c9ed4..afcc5dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -385,15 +385,46 @@ public class FsVolumeImpl implements FsVolumeSpi {
*/
@Override
public long getAvailable() throws IOException {
- long remaining = getCapacity() - getDfsUsed() - reservedForReplicas.get();
- long available = usage.getAvailable() - reserved
- - reservedForReplicas.get();
+ long remaining = getCapacity() - getDfsUsed() - getReservedForReplicas();
+ long available = usage.getAvailable() - getRemainingReserved()
+ - getReservedForReplicas();
if (remaining > available) {
remaining = available;
}
return (remaining > 0) ? remaining : 0;
}
+ long getActualNonDfsUsed() throws IOException {
+ return usage.getUsed() - getDfsUsed();
+ }
+
+ private long getRemainingReserved() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return reserved - actualNonDfsUsed;
+ }
+ return 0L;
+ }
+
+ /**
+ * Unplanned Non-DFS usage, i.e. Extra usage beyond reserved.
+ *
+ * @return
+ * @throws IOException
+ */
+ public long getNonDfsUsed() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return 0L;
+ }
+ return actualNonDfsUsed - reserved;
+ }
+
+ @VisibleForTesting
+ long getDfAvailable() {
+ return usage.getAvailable();
+ }
+
@VisibleForTesting
public long getReservedForReplicas() {
return reservedForReplicas.get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 45cd4ea..af17756 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -720,6 +720,28 @@ public class TestPBHelper {
assertBlockECRecoveryInfoEquals(blkECRecoveryInfo1, iterator.next());
}
+ @Test
+ public void testDataNodeInfoPBHelper() {
+ DatanodeID id = DFSTestUtil.getLocalDatanodeID();
+ DatanodeInfo dnInfos0 = new DatanodeInfo(id);
+ dnInfos0.setCapacity(3500L);
+ dnInfos0.setDfsUsed(1000L);
+ dnInfos0.setNonDfsUsed(2000L);
+ dnInfos0.setRemaining(500L);
+ HdfsProtos.DatanodeInfoProto dnproto = PBHelperClient.convert(dnInfos0);
+ DatanodeInfo dnInfos1 = PBHelperClient.convert(dnproto);
+ compare(dnInfos0, dnInfos1);
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos1.getNonDfsUsed());
+
+ //Testing without nonDfs field
+ HdfsProtos.DatanodeInfoProto.Builder b =
+ HdfsProtos.DatanodeInfoProto.newBuilder();
+ b.setId(PBHelperClient.convert(id)).setCapacity(3500L).setDfsUsed(1000L)
+ .setRemaining(500L);
+ DatanodeInfo dnInfos3 = PBHelperClient.convert(b.build());
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos3.getNonDfsUsed());
+ }
+
private void assertBlockECRecoveryInfoEquals(
BlockECReconstructionInfo blkECRecoveryInfo1,
BlockECReconstructionInfo blkECRecoveryInfo2) {
@@ -768,4 +790,6 @@ public class TestPBHelper {
compare(dnInfos1[i], dnInfos2[i]);
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 1b565ee..5fb3b25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -294,7 +294,7 @@ public class BlockManagerTestUtil {
StorageReport report = new StorageReport(
dns ,false, storage.getCapacity(),
storage.getDfsUsed(), storage.getRemaining(),
- storage.getBlockPoolUsed());
+ storage.getBlockPoolUsed(), 0);
reports.add(report);
}
return reports.toArray(StorageReport.EMPTY_ARRAY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 1f58f99..942569a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1200,7 +1200,7 @@ public class TestBlockManager {
StorageReport report = new StorageReport(
dns, true, storageInfo.getCapacity(),
storageInfo.getDfsUsed(), storageInfo.getRemaining(),
- storageInfo.getBlockPoolUsed());
+ storageInfo.getBlockPoolUsed(), 0L);
reports.add(report);
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index c0c76ad..a082fbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -441,7 +441,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
synchronized StorageReport getStorageReport(String bpid) {
return new StorageReport(dnStorage,
false, getCapacity(), getUsed(), getFree(),
- map.get(bpid).getUsed());
+ map.get(bpid).getUsed(), 0L);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index b8564bb..fcd960a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -72,7 +72,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
@Override
public StorageReport[] getStorageReports(String bpid) throws IOException {
StorageReport[] result = new StorageReport[1];
- result[0] = new StorageReport(storage, false, 0, 0, 0, 0);
+ result[0] = new StorageReport(storage, false, 0, 0, 0, 0, 0);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 796d249..3d4c38c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
@@ -177,4 +178,43 @@ public class TestFsVolumeList {
conf, StorageType.DEFAULT);
assertEquals("", 100L, volume4.getReserved());
}
+
+ @Test
+ public void testNonDfsUsedMetricForVolume() throws Exception {
+ File volDir = new File(baseDir, "volume-0");
+ volDir.mkdirs();
+ /*
+ * Lets have the example.
+ * Capacity - 1000
+ * Reserved - 100
+ * DfsUsed - 200
+ * Actual Non-DfsUsed - 300 -->(expected)
+ * ReservedForReplicas - 50
+ */
+ long diskCapacity = 1000L;
+ long duReserved = 100L;
+ long dfsUsage = 200L;
+ long actualNonDfsUsage = 300L;
+ long reservedForReplicas = 50L;
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, duReserved);
+ FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf,
+ StorageType.DEFAULT);
+ FsVolumeImpl spyVolume = Mockito.spy(volume);
+ // Set Capacity for testing
+ long testCapacity = diskCapacity - duReserved;
+ spyVolume.setCapacityForTesting(testCapacity);
+ // Mock volume.getDfAvailable()
+ long dfAvailable = diskCapacity - dfsUsage - actualNonDfsUsage;
+ Mockito.doReturn(dfAvailable).when(spyVolume).getDfAvailable();
+ // Mock dfsUsage
+ Mockito.doReturn(dfsUsage).when(spyVolume).getDfsUsed();
+ // Mock reservedForReplcas
+ Mockito.doReturn(reservedForReplicas).when(spyVolume)
+ .getReservedForReplicas();
+ Mockito.doReturn(actualNonDfsUsage).when(spyVolume)
+ .getActualNonDfsUsed();
+ long expectedNonDfsUsage =
+ actualNonDfsUsage - duReserved;
+ assertEquals(expectedNonDfsUsage, spyVolume.getNonDfsUsed());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index be2a678..ad336ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -949,7 +949,7 @@ public class NNThroughputBenchmark implements Tool {
// register datanode
// TODO:FEDERATION currently a single block pool is supported
StorageReport[] rep = { new StorageReport(storage, false,
- DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0L) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration, rep,
0L, 0L, 0, 0, 0, null, true).getCommands();
if(cmds != null) {
@@ -998,7 +998,7 @@ public class NNThroughputBenchmark implements Tool {
int replicateBlocks() throws IOException {
// register datanode
StorageReport[] rep = { new StorageReport(storage,
- false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration,
rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
if (cmds != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index 341a149..14b14b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -128,7 +128,7 @@ public class TestDeadDatanode {
// that asks datanode to register again
StorageReport[] rep = { new StorageReport(
new DatanodeStorage(reg.getDatanodeUuid()),
- false, 0, 0, 0, 0) };
+ false, 0, 0, 0, 0, 0) };
DatanodeCommand[] cmd =
dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
assertEquals(1, cmd.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
index 99323f5..38ec9f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
@@ -71,7 +71,6 @@ public class TestNamenodeCapacityReport {
try {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
-
final FSNamesystem namesystem = cluster.getNamesystem();
final DatanodeManager dm = cluster.getNamesystem().getBlockManager(
).getDatanodeManager();
@@ -100,8 +99,9 @@ public class TestNamenodeCapacityReport {
+ " used " + used + " non DFS used " + nonDFSUsed
+ " remaining " + remaining + " perentUsed " + percentUsed
+ " percentRemaining " + percentRemaining);
-
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
configCapacity));
assertTrue(percentRemaining == DFSUtilClient.getPercentRemaining(
@@ -148,7 +148,9 @@ public class TestNamenodeCapacityReport {
assertTrue(configCapacity == diskCapacity - reserved);
// Ensure new total capacity reported excludes the reserved space
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
@@ -160,9 +162,33 @@ public class TestNamenodeCapacityReport {
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentRemaining == ((float)remaining * 100.0f)/(float)configCapacity);
+
+ //Adding testcase for non-dfs used where we need to consider
+ // reserved replica also.
+ final int fileCount = 5;
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ // create streams and hsync to force datastreamers to start
+ DFSOutputStream[] streams = new DFSOutputStream[fileCount];
+ for (int i=0; i < fileCount; i++) {
+ streams[i] = (DFSOutputStream)fs.create(new Path("/f"+i))
+ .getWrappedStream();
+ streams[i].write("1".getBytes());
+ streams[i].hsync();
+ }
+ triggerHeartbeats(cluster.getDataNodes());
+ assertTrue(configCapacity > (namesystem.getCapacityUsed() + namesystem
+ .getCapacityRemaining() + namesystem.getNonDfsUsedSpace()));
+ // There is a chance that nonDFS usage might have slightly due to
+ // testlogs, So assume 1MB other files used within this gap
+ assertTrue(
+ (namesystem.getCapacityUsed() + namesystem.getCapacityRemaining()
+ + namesystem.getNonDfsUsedSpace() + fileCount * fs
+ .getDefaultBlockSize()) - configCapacity < 1 * 1024);
}
finally {
- if (cluster != null) {cluster.shutdown();}
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index c4a9837..9e95650 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -166,7 +166,9 @@ public class TestNameNodeMetrics {
MetricsAsserts.getLongGauge("CapacityRemaining", rb);
long capacityUsedNonDFS =
MetricsAsserts.getLongGauge("CapacityUsedNonDFS", rb);
- assert(capacityUsed + capacityRemaining + capacityUsedNonDFS ==
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assert (capacityUsed + capacityRemaining + capacityUsedNonDFS <=
capacityTotal);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/43] hadoop git commit: YARN-5549.
AMLauncher#createAMContainerLaunchContext() should not log the command to be
launched indiscriminately. (Daniel Templeton via rchiang)
Posted by ae...@apache.org.
YARN-5549. AMLauncher#createAMContainerLaunchContext() should not log the command to be launched indiscriminately. (Daniel Templeton via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/378f624a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/378f624a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/378f624a
Branch: refs/heads/HDFS-7240
Commit: 378f624a392550770d1db33cb4cee3ef7d5facd4
Parents: 5a8c506
Author: Ray Chiang <rc...@apache.org>
Authored: Fri Sep 2 11:07:39 2016 -0700
Committer: Ray Chiang <rc...@apache.org>
Committed: Fri Sep 2 11:14:35 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 12 +++++++++
.../src/main/resources/yarn-default.xml | 13 ++++++++++
.../resourcemanager/amlauncher/AMLauncher.java | 26 ++++++++++++++++----
3 files changed, 46 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 46e3323..86e8a95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -534,6 +534,18 @@ public class YarnConfiguration extends Configuration {
public static final int
DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10;
+ /**
+ * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the
+ * command being executed to the RM log if this property is true. Commands
+ * may contain sensitive information, such as application or service
+ * passwords, making logging the commands a security risk. In cases where
+ * the cluster may be running applications with such commands, this property
+ * should be set to false. Commands are only logged at the debug level.
+ */
+ public static final String RM_AMLAUNCHER_LOG_COMMAND =
+ RM_PREFIX + "amlauncher.log.command";
+ public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false;
+
//RM delegation token related keys
public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e956507..423b78b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -299,6 +299,19 @@
</property>
<property>
+ <description>
+ The resource manager will log all commands being executed to the RM log
+ if this property is true. Commands may contain sensitive information,
+ such as application or service passwords, making logging the commands a
+ security risk. In cases where the cluster may be running applications with
+ such commands this property should be set to false. Commands are only
+ logged at the debug level.
+ </description>
+ <name>yarn.resourcemanager.amlauncher.log.command</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>The class to use as the resource scheduler.</description>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 4aace2c..181463a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
/**
* The launch of the AM itself.
@@ -81,6 +82,7 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
+ private final boolean logCommandLine;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -93,6 +95,9 @@ public class AMLauncher implements Runnable {
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
+ this.logCommandLine =
+ conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND,
+ YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND);
}
private void connect() throws IOException {
@@ -188,11 +193,22 @@ public class AMLauncher implements Runnable {
// Construct the actual Container
ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
- LOG.info("Command to launch container "
- + containerID
- + " : "
- + StringUtils.arrayToString(container.getCommands().toArray(
- new String[0])));
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder message = new StringBuilder("Command to launch container ");
+
+ message.append(containerID).append(" : ");
+
+ if (logCommandLine) {
+ message.append(Joiner.on(",").join(container.getCommands()));
+ } else {
+ message.append("<REDACTED> -- Set ");
+ message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND);
+ message.append(" to true to reenable command logging");
+ }
+
+ LOG.debug(message.toString());
+ }
// Populate the current queue name in the environment variable.
setupQueueNameEnv(container, applicationMasterContext);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/43] hadoop git commit: HADOOP-13549. Eliminate intermediate
buffer for server-side PB encoding. Contributed by Daryn Sharp.
Posted by ae...@apache.org.
HADOOP-13549. Eliminate intermediate buffer for server-side PB encoding. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39d1b1d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39d1b1d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39d1b1d7
Branch: refs/heads/HDFS-7240
Commit: 39d1b1d747b1e325792b897b3264272f32b756a9
Parents: e6fcfe2
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Sep 6 10:01:01 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Sep 6 10:02:42 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 52 ++++++++++++++++++--
1 file changed, 49 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d1b1d7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index ea4f533..531d574 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -119,6 +119,7 @@ import org.apache.htrace.core.Tracer;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
/** An abstract IPC service. IPC calls take a single {@link Writable} as a
@@ -2752,24 +2753,69 @@ public abstract class Server {
private void setupResponse(RpcCall call,
RpcResponseHeaderProto header, Writable rv) throws IOException {
+ final byte[] response;
+ if (rv == null || (rv instanceof RpcWritable.ProtobufWrapper)) {
+ response = setupResponseForProtobuf(header, rv);
+ } else {
+ response = setupResponseForWritable(header, rv);
+ }
+ if (response.length > maxRespSize) {
+ LOG.warn("Large response size " + response.length + " for call "
+ + call.toString());
+ }
+ call.setResponse(ByteBuffer.wrap(response));
+ }
+
+ private byte[] setupResponseForWritable(
+ RpcResponseHeaderProto header, Writable rv) throws IOException {
ResponseBuffer buf = responseBuffer.get().reset();
try {
RpcWritable.wrap(header).writeTo(buf);
if (rv != null) {
RpcWritable.wrap(rv).writeTo(buf);
}
- call.setResponse(ByteBuffer.wrap(buf.toByteArray()));
+ return buf.toByteArray();
} finally {
// Discard a large buf and reset it back to smaller size
// to free up heap.
if (buf.capacity() > maxRespSize) {
- LOG.warn("Large response size " + buf.size() + " for call "
- + call.toString());
buf.setCapacity(INITIAL_RESP_BUF_SIZE);
}
}
}
+
+ // writing to a pre-allocated array is the most efficient way to construct
+ // a protobuf response.
+ private byte[] setupResponseForProtobuf(
+ RpcResponseHeaderProto header, Writable rv) throws IOException {
+ Message payload = (rv != null)
+ ? ((RpcWritable.ProtobufWrapper)rv).getMessage() : null;
+ int length = getDelimitedLength(header);
+ if (payload != null) {
+ length += getDelimitedLength(payload);
+ }
+ byte[] buf = new byte[length + 4];
+ CodedOutputStream cos = CodedOutputStream.newInstance(buf);
+ // the stream only supports little endian ints
+ cos.writeRawByte((byte)((length >>> 24) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 16) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 8) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 0) & 0xFF));
+ cos.writeRawVarint32(header.getSerializedSize());
+ header.writeTo(cos);
+ if (payload != null) {
+ cos.writeRawVarint32(payload.getSerializedSize());
+ payload.writeTo(cos);
+ }
+ return buf;
+ }
+
+ private static int getDelimitedLength(Message message) {
+ int length = message.getSerializedSize();
+ return length + CodedOutputStream.computeRawVarint32Size(length);
+ }
+
/**
* Setup response for the IPC Call on Fatal Error from a
* client that is using old version of Hadoop.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/43] hadoop git commit: HADOOP-13547. Optimize IPC client protobuf
decoding. Contributed by Daryn Sharp.
Posted by ae...@apache.org.
HADOOP-13547. Optimize IPC client protobuf decoding. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23abb09c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23abb09c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23abb09c
Branch: refs/heads/HDFS-7240
Commit: 23abb09c1f979d8c18ece81e32630a35ed569399
Parents: 05f5c0f
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 2 11:03:18 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 2 11:03:18 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Client.java | 77 +++---
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 241 +++----------------
.../org/apache/hadoop/ipc/ResponseBuffer.java | 12 +-
.../java/org/apache/hadoop/ipc/RpcWritable.java | 20 +-
.../apache/hadoop/security/SaslRpcClient.java | 45 ++--
5 files changed, 102 insertions(+), 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 183bad4..567b932 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.ipc;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.CodedOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,13 +30,11 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
-import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
import org.apache.hadoop.ipc.RPC.RpcKind;
import org.apache.hadoop.ipc.Server.AuthProtocol;
import org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
@@ -54,7 +51,6 @@ import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ProtoUtil;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.AsyncGet;
@@ -65,6 +61,7 @@ import javax.net.SocketFactory;
import javax.security.sasl.Sasl;
import java.io.*;
import java.net.*;
+import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.*;
import java.util.Map.Entry;
@@ -429,7 +426,7 @@ public class Client implements AutoCloseable {
private final boolean doPing; //do we need to send ping message
private final int pingInterval; // how often sends ping to the server
private final int soTimeout; // used by ipc ping and rpc timeout
- private ByteArrayOutputStream pingRequest; // ping message
+ private ResponseBuffer pingRequest; // ping message
// currently active calls
private Hashtable<Integer, Call> calls = new Hashtable<Integer, Call>();
@@ -459,7 +456,7 @@ public class Client implements AutoCloseable {
this.doPing = remoteId.getDoPing();
if (doPing) {
// construct a RPC header with the callId as the ping callId
- pingRequest = new ByteArrayOutputStream();
+ pingRequest = new ResponseBuffer();
RpcRequestHeaderProto pingHeader = ProtoUtil
.makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
OperationProto.RPC_FINAL_PACKET, PING_CALL_ID,
@@ -979,12 +976,10 @@ public class Client implements AutoCloseable {
.makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
OperationProto.RPC_FINAL_PACKET, CONNECTION_CONTEXT_CALL_ID,
RpcConstants.INVALID_RETRY_COUNT, clientId);
- RpcRequestMessageWrapper request =
- new RpcRequestMessageWrapper(connectionContextHeader, message);
-
- // Write out the packet length
- out.writeInt(request.getLength());
- request.write(out);
+ final ResponseBuffer buf = new ResponseBuffer();
+ connectionContextHeader.writeDelimitedTo(buf);
+ message.writeDelimitedTo(buf);
+ buf.writeTo(out);
}
/* wait till someone signals us to start reading RPC response or
@@ -1030,7 +1025,6 @@ public class Client implements AutoCloseable {
if ( curTime - lastActivity.get() >= pingInterval) {
lastActivity.set(curTime);
synchronized (out) {
- out.writeInt(pingRequest.size());
pingRequest.writeTo(out);
out.flush();
}
@@ -1085,12 +1079,13 @@ public class Client implements AutoCloseable {
// 2) RpcRequest
//
// Items '1' and '2' are prepared here.
- final DataOutputBuffer d = new DataOutputBuffer();
RpcRequestHeaderProto header = ProtoUtil.makeRpcRequestHeader(
call.rpcKind, OperationProto.RPC_FINAL_PACKET, call.id, call.retry,
clientId);
- header.writeDelimitedTo(d);
- call.rpcRequest.write(d);
+
+ final ResponseBuffer buf = new ResponseBuffer();
+ header.writeDelimitedTo(buf);
+ RpcWritable.wrap(call.rpcRequest).writeTo(buf);
synchronized (sendRpcRequestLock) {
Future<?> senderFuture = sendParamsExecutor.submit(new Runnable() {
@@ -1101,14 +1096,10 @@ public class Client implements AutoCloseable {
if (shouldCloseConnection.get()) {
return;
}
-
- if (LOG.isDebugEnabled())
+ if (LOG.isDebugEnabled()) {
LOG.debug(getName() + " sending #" + call.id);
-
- byte[] data = d.getData();
- int totalLength = d.getLength();
- out.writeInt(totalLength); // Total Length
- out.write(data, 0, totalLength);// RpcRequestHeader + RpcRequest
+ }
+ buf.writeTo(out); // RpcRequestHeader + RpcRequest
out.flush();
}
} catch (IOException e) {
@@ -1119,7 +1110,7 @@ public class Client implements AutoCloseable {
} finally {
//the buffer is just an in-memory buffer, but it is still polite to
// close early
- IOUtils.closeStream(d);
+ IOUtils.closeStream(buf);
}
}
});
@@ -1151,12 +1142,13 @@ public class Client implements AutoCloseable {
try {
int totalLen = in.readInt();
- RpcResponseHeaderProto header =
- RpcResponseHeaderProto.parseDelimitedFrom(in);
- checkResponse(header);
+ ByteBuffer bb = ByteBuffer.allocate(totalLen);
+ in.readFully(bb.array());
- int headerLen = header.getSerializedSize();
- headerLen += CodedOutputStream.computeRawVarint32Size(headerLen);
+ RpcWritable.Buffer packet = RpcWritable.Buffer.wrap(bb);
+ RpcResponseHeaderProto header =
+ packet.getValue(RpcResponseHeaderProto.getDefaultInstance());
+ checkResponse(header);
int callId = header.getCallId();
if (LOG.isDebugEnabled())
@@ -1164,28 +1156,15 @@ public class Client implements AutoCloseable {
RpcStatusProto status = header.getStatus();
if (status == RpcStatusProto.SUCCESS) {
- Writable value = ReflectionUtils.newInstance(valueClass, conf);
- value.readFields(in); // read value
+ Writable value = packet.newInstance(valueClass, conf);
final Call call = calls.remove(callId);
call.setRpcResponse(value);
-
- // verify that length was correct
- // only for ProtobufEngine where len can be verified easily
- if (call.getRpcResponse() instanceof ProtobufRpcEngine.RpcWrapper) {
- ProtobufRpcEngine.RpcWrapper resWrapper =
- (ProtobufRpcEngine.RpcWrapper) call.getRpcResponse();
- if (totalLen != headerLen + resWrapper.getLength()) {
- throw new RpcClientException(
- "RPC response length mismatch on rpc success");
- }
- }
- } else { // Rpc Request failed
- // Verify that length was correct
- if (totalLen != headerLen) {
- throw new RpcClientException(
- "RPC response length mismatch on rpc error");
- }
-
+ }
+ // verify that packet length was correct
+ if (packet.remaining() > 0) {
+ throw new RpcClientException("RPC response length mismatch");
+ }
+ if (status != RpcStatusProto.SUCCESS) { // Rpc Request failed
final String exceptionClassName = header.hasExceptionClassName() ?
header.getExceptionClassName() :
"ServerDidNotSetExceptionClassName";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index eb30aa2..83e4b9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -27,29 +27,22 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.DataOutputOutputStream;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.RPC.RpcInvoker;
import org.apache.hadoop.ipc.RpcWritable;
import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestHeaderProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.ProtoUtil;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.AsyncGet;
import org.apache.htrace.core.TraceScope;
import org.apache.htrace.core.Tracer;
import javax.net.SocketFactory;
-import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
-import java.io.OutputStream;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.net.InetSocketAddress;
@@ -146,7 +139,7 @@ public class ProtobufRpcEngine implements RpcEngine {
private Invoker(Class<?> protocol, Client.ConnectionId connId,
Configuration conf, SocketFactory factory) {
this.remoteId = connId;
- this.client = CLIENTS.getClient(conf, factory, RpcResponseWrapper.class);
+ this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class);
this.protocolName = RPC.getProtocolName(protocol);
this.clientProtocolVersion = RPC
.getProtocolVersion(protocol);
@@ -193,7 +186,7 @@ public class ProtobufRpcEngine implements RpcEngine {
* the server.
*/
@Override
- public Object invoke(Object proxy, final Method method, Object[] args)
+ public Message invoke(Object proxy, final Method method, Object[] args)
throws ServiceException {
long startTime = 0;
if (LOG.isDebugEnabled()) {
@@ -228,11 +221,11 @@ public class ProtobufRpcEngine implements RpcEngine {
}
- Message theRequest = (Message) args[1];
- final RpcResponseWrapper val;
+ final Message theRequest = (Message) args[1];
+ final RpcWritable.Buffer val;
try {
- val = (RpcResponseWrapper) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
- new RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId,
+ val = (RpcWritable.Buffer) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ new RpcProtobufRequest(rpcRequestHeader, theRequest), remoteId,
fallbackToSimpleAuth);
} catch (Throwable e) {
@@ -256,7 +249,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (Client.isAsynchronousMode()) {
- final AsyncGet<RpcResponseWrapper, IOException> arr
+ final AsyncGet<RpcWritable.Buffer, IOException> arr
= Client.getAsyncRpcResponse();
final AsyncGet<Message, Exception> asyncGet
= new AsyncGet<Message, Exception>() {
@@ -278,7 +271,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
private Message getReturnMessage(final Method method,
- final RpcResponseWrapper rrw) throws ServiceException {
+ final RpcWritable.Buffer buf) throws ServiceException {
Message prototype = null;
try {
prototype = getReturnProtoType(method);
@@ -287,8 +280,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
Message returnMessage;
try {
- returnMessage = prototype.newBuilderForType()
- .mergeFrom(rrw.theResponseRead).build();
+ returnMessage = buf.getValue(prototype.getDefaultInstanceForType());
if (LOG.isTraceEnabled()) {
LOG.trace(Thread.currentThread().getId() + ": Response <- " +
@@ -329,201 +321,12 @@ public class ProtobufRpcEngine implements RpcEngine {
}
}
- interface RpcWrapper extends Writable {
- int getLength();
- }
- /**
- * Wrapper for Protocol Buffer Requests
- *
- * Note while this wrapper is writable, the request on the wire is in
- * Protobuf. Several methods on {@link org.apache.hadoop.ipc.Server and RPC}
- * use type Writable as a wrapper to work across multiple RpcEngine kinds.
- */
- private static abstract class RpcMessageWithHeader<T extends GeneratedMessage>
- implements RpcWrapper {
- T requestHeader;
- Message theRequest; // for clientSide, the request is here
- byte[] theRequestRead; // for server side, the request is here
-
- public RpcMessageWithHeader() {
- }
-
- public RpcMessageWithHeader(T requestHeader, Message theRequest) {
- this.requestHeader = requestHeader;
- this.theRequest = theRequest;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- OutputStream os = DataOutputOutputStream.constructOutputStream(out);
-
- ((Message)requestHeader).writeDelimitedTo(os);
- theRequest.writeDelimitedTo(os);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- requestHeader = parseHeaderFrom(readVarintBytes(in));
- theRequestRead = readMessageRequest(in);
- }
-
- abstract T parseHeaderFrom(byte[] bytes) throws IOException;
-
- byte[] readMessageRequest(DataInput in) throws IOException {
- return readVarintBytes(in);
- }
-
- private static byte[] readVarintBytes(DataInput in) throws IOException {
- final int length = ProtoUtil.readRawVarint32(in);
- final byte[] bytes = new byte[length];
- in.readFully(bytes);
- return bytes;
- }
-
- public T getMessageHeader() {
- return requestHeader;
- }
-
- public byte[] getMessageBytes() {
- return theRequestRead;
- }
-
- @Override
- public int getLength() {
- int headerLen = requestHeader.getSerializedSize();
- int reqLen;
- if (theRequest != null) {
- reqLen = theRequest.getSerializedSize();
- } else if (theRequestRead != null ) {
- reqLen = theRequestRead.length;
- } else {
- throw new IllegalArgumentException(
- "getLength on uninitialized RpcWrapper");
- }
- return CodedOutputStream.computeRawVarint32Size(headerLen) + headerLen
- + CodedOutputStream.computeRawVarint32Size(reqLen) + reqLen;
- }
- }
-
- private static class RpcRequestWrapper
- extends RpcMessageWithHeader<RequestHeaderProto> {
- @SuppressWarnings("unused")
- public RpcRequestWrapper() {}
-
- public RpcRequestWrapper(
- RequestHeaderProto requestHeader, Message theRequest) {
- super(requestHeader, theRequest);
- }
-
- @Override
- RequestHeaderProto parseHeaderFrom(byte[] bytes) throws IOException {
- return RequestHeaderProto.parseFrom(bytes);
- }
-
- @Override
- public String toString() {
- return requestHeader.getDeclaringClassProtocolName() + "." +
- requestHeader.getMethodName();
- }
- }
-
- @InterfaceAudience.LimitedPrivate({"RPC"})
- public static class RpcRequestMessageWrapper
- extends RpcMessageWithHeader<RpcRequestHeaderProto> {
- public RpcRequestMessageWrapper() {}
-
- public RpcRequestMessageWrapper(
- RpcRequestHeaderProto requestHeader, Message theRequest) {
- super(requestHeader, theRequest);
- }
-
- @Override
- RpcRequestHeaderProto parseHeaderFrom(byte[] bytes) throws IOException {
- return RpcRequestHeaderProto.parseFrom(bytes);
- }
- }
-
- @InterfaceAudience.LimitedPrivate({"RPC"})
- public static class RpcResponseMessageWrapper
- extends RpcMessageWithHeader<RpcResponseHeaderProto> {
- public RpcResponseMessageWrapper() {}
-
- public RpcResponseMessageWrapper(
- RpcResponseHeaderProto responseHeader, Message theRequest) {
- super(responseHeader, theRequest);
- }
-
- @Override
- byte[] readMessageRequest(DataInput in) throws IOException {
- // error message contain no message body
- switch (requestHeader.getStatus()) {
- case ERROR:
- case FATAL:
- return null;
- default:
- return super.readMessageRequest(in);
- }
- }
-
- @Override
- RpcResponseHeaderProto parseHeaderFrom(byte[] bytes) throws IOException {
- return RpcResponseHeaderProto.parseFrom(bytes);
- }
- }
-
- /**
- * Wrapper for Protocol Buffer Responses
- *
- * Note while this wrapper is writable, the request on the wire is in
- * Protobuf. Several methods on {@link org.apache.hadoop.ipc.Server and RPC}
- * use type Writable as a wrapper to work across multiple RpcEngine kinds.
- */
- @InterfaceAudience.LimitedPrivate({"RPC"}) // temporarily exposed
- public static class RpcResponseWrapper implements RpcWrapper {
- Message theResponse; // for senderSide, the response is here
- byte[] theResponseRead; // for receiver side, the response is here
-
- public RpcResponseWrapper() {
- }
-
- public RpcResponseWrapper(Message message) {
- this.theResponse = message;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- OutputStream os = DataOutputOutputStream.constructOutputStream(out);
- theResponse.writeDelimitedTo(os);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- int length = ProtoUtil.readRawVarint32(in);
- theResponseRead = new byte[length];
- in.readFully(theResponseRead);
- }
-
- @Override
- public int getLength() {
- int resLen;
- if (theResponse != null) {
- resLen = theResponse.getSerializedSize();
- } else if (theResponseRead != null ) {
- resLen = theResponseRead.length;
- } else {
- throw new IllegalArgumentException(
- "getLength on uninitialized RpcWrapper");
- }
- return CodedOutputStream.computeRawVarint32Size(resLen) + resLen;
- }
- }
-
@VisibleForTesting
@InterfaceAudience.Private
@InterfaceStability.Unstable
static Client getClient(Configuration conf) {
return CLIENTS.getClient(conf, SocketFactory.getDefault(),
- RpcResponseWrapper.class);
+ RpcWritable.Buffer.class);
}
@@ -691,16 +494,30 @@ public class ProtobufRpcEngine implements RpcEngine {
// which uses the rpc header. in the normal case we want to defer decoding
// the rpc header until needed by the rpc engine.
static class RpcProtobufRequest extends RpcWritable.Buffer {
- private RequestHeaderProto lazyHeader;
+ private volatile RequestHeaderProto requestHeader;
+ private Message payload;
public RpcProtobufRequest() {
}
- synchronized RequestHeaderProto getRequestHeader() throws IOException {
- if (lazyHeader == null) {
- lazyHeader = getValue(RequestHeaderProto.getDefaultInstance());
+ RpcProtobufRequest(RequestHeaderProto header, Message payload) {
+ this.requestHeader = header;
+ this.payload = payload;
+ }
+
+ RequestHeaderProto getRequestHeader() throws IOException {
+ if (getByteBuffer() != null && requestHeader == null) {
+ requestHeader = getValue(RequestHeaderProto.getDefaultInstance());
+ }
+ return requestHeader;
+ }
+
+ @Override
+ public void writeTo(ResponseBuffer out) throws IOException {
+ requestHeader.writeDelimitedTo(out);
+ if (payload != null) {
+ payload.writeDelimitedTo(out);
}
- return lazyHeader;
}
// this is used by htrace to name the span.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
index ac96a24..a789d83 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
@@ -27,8 +27,14 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
@InterfaceAudience.Private
-class ResponseBuffer extends DataOutputStream {
- ResponseBuffer(int capacity) {
+/** generates byte-length framed buffers. */
+public class ResponseBuffer extends DataOutputStream {
+
+ public ResponseBuffer() {
+ this(1024);
+ }
+
+ public ResponseBuffer(int capacity) {
super(new FramedBuffer(capacity));
}
@@ -39,7 +45,7 @@ class ResponseBuffer extends DataOutputStream {
return buf;
}
- void writeTo(OutputStream out) throws IOException {
+ public void writeTo(OutputStream out) throws IOException {
getFramedBuffer().writeTo(out);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
index 5125939..54fb98e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
@@ -24,7 +24,6 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -34,6 +33,7 @@ import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
+// note anything marked public is solely for access by SaslRpcClient
@InterfaceAudience.Private
public abstract class RpcWritable implements Writable {
@@ -99,6 +99,10 @@ public abstract class RpcWritable implements Writable {
this.message = message;
}
+ Message getMessage() {
+ return message;
+ }
+
@Override
void writeTo(ResponseBuffer out) throws IOException {
int length = message.getSerializedSize();
@@ -128,11 +132,13 @@ public abstract class RpcWritable implements Writable {
}
}
- // adapter to allow decoding of writables and protobufs from a byte buffer.
- static class Buffer extends RpcWritable {
+ /**
+ * adapter to allow decoding of writables and protobufs from a byte buffer.
+ */
+ public static class Buffer extends RpcWritable {
private ByteBuffer bb;
- static Buffer wrap(ByteBuffer bb) {
+ public static Buffer wrap(ByteBuffer bb) {
return new Buffer(bb);
}
@@ -142,6 +148,10 @@ public abstract class RpcWritable implements Writable {
this.bb = bb;
}
+ ByteBuffer getByteBuffer() {
+ return bb;
+ }
+
@Override
void writeTo(ResponseBuffer out) throws IOException {
out.ensureCapacity(bb.remaining());
@@ -177,7 +187,7 @@ public abstract class RpcWritable implements Writable {
return RpcWritable.wrap(value).readFrom(bb);
}
- int remaining() {
+ public int remaining() {
return bb.remaining();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index c360937..60ae3b0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -53,11 +53,11 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.GlobPattern;
-import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
-import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcResponseMessageWrapper;
import org.apache.hadoop.ipc.RPC.RpcKind;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.ResponseBuffer;
import org.apache.hadoop.ipc.RpcConstants;
+import org.apache.hadoop.ipc.RpcWritable;
import org.apache.hadoop.ipc.Server.AuthProtocol;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto.OperationProto;
@@ -368,11 +368,13 @@ public class SaslRpcClient {
// loop until sasl is complete or a rpc error occurs
boolean done = false;
do {
- int totalLen = inStream.readInt();
- RpcResponseMessageWrapper responseWrapper =
- new RpcResponseMessageWrapper();
- responseWrapper.readFields(inStream);
- RpcResponseHeaderProto header = responseWrapper.getMessageHeader();
+ int rpcLen = inStream.readInt();
+ ByteBuffer bb = ByteBuffer.allocate(rpcLen);
+ inStream.readFully(bb.array());
+
+ RpcWritable.Buffer saslPacket = RpcWritable.Buffer.wrap(bb);
+ RpcResponseHeaderProto header =
+ saslPacket.getValue(RpcResponseHeaderProto.getDefaultInstance());
switch (header.getStatus()) {
case ERROR: // might get a RPC error during
case FATAL:
@@ -380,15 +382,14 @@ public class SaslRpcClient {
header.getErrorMsg());
default: break;
}
- if (totalLen != responseWrapper.getLength()) {
- throw new SaslException("Received malformed response length");
- }
-
if (header.getCallId() != AuthProtocol.SASL.callId) {
throw new SaslException("Non-SASL response during negotiation");
}
RpcSaslProto saslMessage =
- RpcSaslProto.parseFrom(responseWrapper.getMessageBytes());
+ saslPacket.getValue(RpcSaslProto.getDefaultInstance());
+ if (saslPacket.remaining() > 0) {
+ throw new SaslException("Received malformed response length");
+ }
// handle sasl negotiation process
RpcSaslProto.Builder response = null;
switch (saslMessage.getState()) {
@@ -452,16 +453,16 @@ public class SaslRpcClient {
return authMethod;
}
- private void sendSaslMessage(DataOutputStream out, RpcSaslProto message)
+ private void sendSaslMessage(OutputStream out, RpcSaslProto message)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Sending sasl message "+message);
}
- RpcRequestMessageWrapper request =
- new RpcRequestMessageWrapper(saslHeader, message);
- out.writeInt(request.getLength());
- request.write(out);
- out.flush();
+ ResponseBuffer buf = new ResponseBuffer();
+ saslHeader.writeDelimitedTo(buf);
+ message.writeDelimitedTo(buf);
+ buf.writeTo(out);
+ out.flush();
}
/**
@@ -634,12 +635,8 @@ public class SaslRpcClient {
.setState(SaslState.WRAP)
.setToken(ByteString.copyFrom(buf, 0, buf.length))
.build();
- RpcRequestMessageWrapper request =
- new RpcRequestMessageWrapper(saslHeader, saslMessage);
- DataOutputStream dob = new DataOutputStream(out);
- dob.writeInt(request.getLength());
- request.write(dob);
- }
+ sendSaslMessage(out, saslMessage);
+ }
}
/** Release resources used by wrapped saslClient */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/43] hadoop git commit: HADOOP-13541 explicitly declare the Joda
time version S3A depends on. Contributed by Stevel Loughran
Posted by ae...@apache.org.
HADOOP-13541 explicitly declare the Joda time version S3A depends on. Contributed by Stevel Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7fdfcd8a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7fdfcd8a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7fdfcd8a
Branch: refs/heads/HDFS-7240
Commit: 7fdfcd8a6c9e2dd9b0fb6d4196bc371f6f9a676c
Parents: c0e492e
Author: Steve Loughran <st...@apache.org>
Authored: Wed Sep 7 12:19:05 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Sep 7 12:25:23 2016 +0100
----------------------------------------------------------------------
hadoop-project/pom.xml | 7 +++++++
hadoop-tools/hadoop-aws/pom.xml | 4 ++++
2 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fdfcd8a/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 65e9672..d9a01a0 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -88,6 +88,7 @@
<tomcat.version>6.0.44</tomcat.version>
<guice.version>4.0</guice.version>
+ <joda-time.version>2.9.4</joda-time.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
@@ -1037,6 +1038,12 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda-time.version}</version>
+ </dependency>
+
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fdfcd8a/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 13dcdf1..49b0379 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -286,6 +286,10 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-sts</artifactId>
<scope>test</scope>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/43] hadoop git commit: HDFS-6962. ACL inheritance conflicts with
umaskmode. Contributed by Chris Nauroth.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
new file mode 100644
index 0000000..7e9ace1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
@@ -0,0 +1,1075 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="testConf.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Normal mode is test. To run just the commands and dump the output
+ to the log, set it to nocompare -->
+ <mode>test</mode>
+
+ <!-- Comparator types:
+ ExactComparator
+ SubstringComparator
+ RegexpComparator
+ TokenComparator
+ -->
+ <tests>
+ <!-- Tests for setfacl and getfacl-->
+ <test>
+ <description>getfacl: basic permissions</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: basic permissions for directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add an ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r-- /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add multiple ACLs at once</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove an ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,user:charlie:r-x /file1</command>
+ <command>-fs NAMENODE -setfacl -x user:bob /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!bob)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add minimal default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user::rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default\:mask)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : try adding default ACL to file</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>setfacl: Invalid ACL: only directories may have a default ACL</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove one default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -x default:user:charlie /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default:user:charlie).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -k /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all but base ACLs for a directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -b /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!charlie).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!admin).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all but base ACLs for a file</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /file1</command>
+ <command>-fs NAMENODE -setfacl -b /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!charlie).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!admin).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : check inherit default ACL to file</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file</command>
+ <command>-fs NAMENODE -getfacl /dir1/file</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1/file</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : check inherit default ACL to dir</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -getfacl /dir1/dir2</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1/dir2</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rwx$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl -R : recursive</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -m user:user1:r-x,group:users:rwx /dir1/dir2</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:user1:r-x#LF#group::r-x#LF#group:users:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl -R : recursive</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl --set : Set full set of ACLs</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl --set user::rw-,group::r--,other::r--,user:user1:r-x,group:users:rw- /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:user1:r-x#LF#group::r--#LF#group:users:rw-#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl -x mask : remove mask entry along with other ACL entries</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -x mask::,user:charlie,group:admin /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#group::r-x#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: only default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: effective permissions</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:rwx,group::-wx,group:sales:rwx,mask::r-x,default:user:charlie:rwx,default:group::r-x,default:group:sales:rwx,default:mask::rw- /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user:charlie:rwx\t#effective:r-x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group::-wx\t#effective:--x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group:sales:rwx\t#effective:r-x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:user:charlie:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:group::r-x\t#effective:r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:group:sales:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>ls: display extended acl marker</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:rwx,group::-wx,group:sales:rwx,mask::r-x,default:user:charlie:rwx,default:group::r-x,default:group:sales:rwx,default:mask::rw- /dir1/dir2</command>
+ <command>-fs NAMENODE -ls /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^drwxr-xr-x\+( )*-( )*USERNAME( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir1/dir2</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive modify entries with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:charlie:rwx,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive remove entries with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:bob:rwx,user:charlie:rwx,default:user:bob:rwx,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -R -x user:bob,default:user:bob /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive set with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R --set user::rwx,user:charlie:rwx,group::r-x,other::r-x,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>copyFromLocal: copying file into a directory with a default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:rwx /dir1</command>
+ <command>-fs NAMENODE -copyFromLocal CLITEST_DATA/data1k /dir1/data1k</command>
+ <command>-fs NAMENODE -getfacl /dir1/data1k</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# file: /dir1/data1k$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# owner: USERNAME$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# group: supergroup$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user:charlie:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group::r-x\t#effective:r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^other::r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ </tests>
+</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/43] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by ae...@apache.org.
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f73cd7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f73cd7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f73cd7f
Branch: refs/heads/HDFS-7240
Commit: 0f73cd7ff6327a3b263af56cb84c69210177e909
Parents: 6692127 35c5943
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Sep 8 19:34:51 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Sep 8 19:34:51 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/Path.java | 20 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 15 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 4 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../apache/hadoop/util/AutoCloseableLock.java | 28 +-
.../org/apache/hadoop/util/DataChecksum.java | 2 +-
.../hadoop/util/NodeHealthScriptRunner.java | 3 +-
.../java/org/apache/hadoop/fs/TestPath.java | 20 +
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 +-
.../hadoop/ipc/TestMultipleProtocolServer.java | 236 ++++++++-
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 +
.../apache/hadoop/ipc/TestRPCCompatibility.java | 242 +++++++++-
.../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 +-
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 74 +--
.../hadoop/security/TestDoAsEffectiveUser.java | 291 +++++++----
.../security/TestUserGroupInformation.java | 28 +-
.../hadoop/util/TestNodeHealthScriptRunner.java | 7 -
.../hadoop-common/src/test/proto/test.proto | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 68 ++-
.../hadoop/hdfs/DFSStripedInputStream.java | 24 +-
.../hadoop/hdfs/util/StripedBlockUtil.java | 177 ++++---
.../hadoop/hdfs/web/URLConnectionFactory.java | 1 +
.../src/main/native/libhdfs/hdfs.c | 10 +-
.../hadoop-hdfs/src/main/bin/hdfs | 2 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../apache/hadoop/hdfs/InstrumentedLock.java | 185 +++++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 10 +-
.../diskbalancer/command/CancelCommand.java | 23 +-
.../server/diskbalancer/command/Command.java | 6 +-
.../diskbalancer/command/ExecuteCommand.java | 15 +-
.../diskbalancer/command/HelpCommand.java | 22 +-
.../diskbalancer/command/PlanCommand.java | 63 +--
.../diskbalancer/command/QueryCommand.java | 19 +-
.../diskbalancer/command/ReportCommand.java | 18 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +
.../apache/hadoop/hdfs/tools/DiskBalancer.java | 482 -------------------
.../hadoop/hdfs/tools/DiskBalancerCLI.java | 482 +++++++++++++++++++
.../FileDistributionCalculator.java | 20 +-
.../FileDistributionVisitor.java | 28 +-
.../offlineImageViewer/OfflineImageViewer.java | 116 ++---
.../OfflineImageViewerPB.java | 78 +--
.../src/main/resources/hdfs-default.xml | 30 +-
.../src/site/markdown/HDFSCommands.md | 4 +
.../src/site/markdown/HdfsImageViewer.md | 13 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 121 ++++-
.../hadoop/hdfs/TestInstrumentedLock.java | 166 +++++++
.../TestClientProtocolWithDelegationToken.java | 119 +++++
.../command/TestDiskBalancerCommand.java | 16 +-
.../TestOfflineImageViewer.java | 24 +-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 22 +-
.../mapreduce/v2/hs/server/HSAdminServer.java | 3 +
54 files changed, 2361 insertions(+), 1139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f73cd7f/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/43] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by ae...@apache.org.
Add CHANGES and release notes for 3.0.0-alpha1 to site
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f388a209
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f388a209
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f388a209
Branch: refs/heads/HDFS-7240
Commit: f388a209431c51e4062cc4a947274220870558a7
Parents: d152557
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Sep 6 09:51:48 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Sep 6 09:51:55 2016 -0700
----------------------------------------------------------------------
.../3.0.0-alpha1/CHANGES.3.0.0-alpha1.md | 3712 ++++++++++++++++++
.../3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md | 1854 +++++++++
2 files changed, 5566 insertions(+)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/43] hadoop git commit: HADOOP-13519. Make Path Serializable.
Contributed by Steve Loughran
Posted by ae...@apache.org.
HADOOP-13519. Make Path Serializable. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1192781a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1192781a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1192781a
Branch: refs/heads/HDFS-7240
Commit: 1192781a78e7fcfcbf01b15de5d5a1f22d982e0d
Parents: b6d839a
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Sep 8 17:46:12 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Sep 8 17:46:12 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/Path.java | 20 +++++++++++++++++++-
.../java/org/apache/hadoop/fs/TestPath.java | 20 ++++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1192781a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
index f18a675..252b3cc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.fs;
import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInputValidation;
+import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.regex.Pattern;
@@ -37,7 +40,7 @@ import org.apache.hadoop.conf.Configuration;
@Stringable
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class Path implements Comparable {
+public class Path implements Comparable, Serializable, ObjectInputValidation {
/**
* The directory separator, a slash.
@@ -66,6 +69,8 @@ public class Path implements Comparable {
private static final Pattern HAS_DRIVE_LETTER_SPECIFIER =
Pattern.compile("^/?[a-zA-Z]:");
+ private static final long serialVersionUID = 0xad00f;
+
private URI uri; // a hierarchical uri
/**
@@ -565,4 +570,17 @@ public class Path implements Comparable {
}
return new Path(newUri);
}
+
+ /**
+ * Validate the contents of a deserialized Path, so as
+ * to defend against malicious object streams.
+ * @throws InvalidObjectException if there's no URI
+ */
+ @Override
+ public void validateObject() throws InvalidObjectException {
+ if (uri == null) {
+ throw new InvalidObjectException("No URI in deserialized Path");
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1192781a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
index dc48a10..5123526 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java
@@ -17,9 +17,14 @@
*/
package org.apache.hadoop.fs;
+import org.junit.Assert;
import org.junit.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
@@ -506,4 +511,19 @@ public class TestPath {
assertFalse(Path.isWindowsAbsolutePath("C:test", false));
assertFalse(Path.isWindowsAbsolutePath("/C:test", true));
}
+
+ @Test(timeout = 30000)
+ public void testSerDeser() throws Throwable {
+ Path source = new Path("hdfs://localhost:4040/scratch");
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(256);
+ try(ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+ oos.writeObject(source);
+ }
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ try (ObjectInputStream ois = new ObjectInputStream(bais)) {
+ Path deser = (Path) ois.readObject();
+ Assert.assertEquals(source, deser);
+ }
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/43] hadoop git commit: HDFS-8901. Use ByteBuffer in striping
positional read. Contributed by Sammi Chen and Kai Zheng.
Posted by ae...@apache.org.
HDFS-8901. Use ByteBuffer in striping positional read. Contributed by Sammi Chen and Kai Zheng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/401db4fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/401db4fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/401db4fc
Branch: refs/heads/HDFS-7240
Commit: 401db4fc65140979fe7665983e36905e886df971
Parents: 20a20c2
Author: Zhe Zhang <zh...@apache.org>
Authored: Thu Sep 8 11:54:33 2016 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Thu Sep 8 11:54:33 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/util/DataChecksum.java | 2 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 68 ++++---
.../hadoop/hdfs/DFSStripedInputStream.java | 24 ++-
.../hadoop/hdfs/util/StripedBlockUtil.java | 177 ++++++++++---------
.../hadoop/hdfs/TestDFSStripedInputStream.java | 121 ++++++++++++-
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 22 +--
6 files changed, 281 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401db4fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index 3a53ed9..6982a92 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -304,7 +304,7 @@ public class DataChecksum implements Checksum {
bytesPerChecksum, checksums.array(), crcsOffset, fileName, basePos);
return;
}
- if (NativeCrc32.isAvailable()) {
+ if (NativeCrc32.isAvailable() && data.isDirect()) {
NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data,
fileName, basePos);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401db4fc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 7a10ba4..31fa897 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -533,7 +533,8 @@ public class DFSInputStream extends FSInputStream
* Open a DataInputStream to a DataNode so that it can be read from.
* We get block ID and the IDs of the destinations at startup, from the namenode.
*/
- private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
+ private synchronized DatanodeInfo blockSeekTo(long target)
+ throws IOException {
if (target >= getFileLength()) {
throw new IOException("Attempted to read past end of file");
}
@@ -962,14 +963,14 @@ public class DFSInputStream extends FSInputStream
}
protected void fetchBlockByteRange(LocatedBlock block, long start, long end,
- byte[] buf, int offset, CorruptedBlocks corruptedBlocks)
+ ByteBuffer buf, CorruptedBlocks corruptedBlocks)
throws IOException {
block = refreshLocatedBlock(block);
while (true) {
DNAddrPair addressPair = chooseDataNode(block, null);
try {
actualGetFromOneDataNode(addressPair, block, start, end,
- buf, offset, corruptedBlocks);
+ buf, corruptedBlocks);
return;
} catch (IOException e) {
checkInterrupted(e); // check if the read has been interrupted
@@ -988,12 +989,10 @@ public class DFSInputStream extends FSInputStream
return new Callable<ByteBuffer>() {
@Override
public ByteBuffer call() throws Exception {
- byte[] buf = bb.array();
- int offset = bb.position();
try (TraceScope ignored = dfsClient.getTracer().
newScope("hedgedRead" + hedgedReadId, parentSpanId)) {
- actualGetFromOneDataNode(datanode, block, start, end, buf,
- offset, corruptedBlocks);
+ actualGetFromOneDataNode(datanode, block, start, end, bb,
+ corruptedBlocks);
return bb;
}
}
@@ -1007,13 +1006,12 @@ public class DFSInputStream extends FSInputStream
* @param block the located block containing the requested data
* @param startInBlk the startInBlk offset of the block
* @param endInBlk the endInBlk offset of the block
- * @param buf the given byte array into which the data is read
- * @param offset the offset in buf
+ * @param buf the given byte buffer into which the data is read
* @param corruptedBlocks map recording list of datanodes with corrupted
* block replica
*/
void actualGetFromOneDataNode(final DNAddrPair datanode, LocatedBlock block,
- final long startInBlk, final long endInBlk, byte[] buf, int offset,
+ final long startInBlk, final long endInBlk, ByteBuffer buf,
CorruptedBlocks corruptedBlocks)
throws IOException {
DFSClientFaultInjector.get().startFetchFromDatanode();
@@ -1031,7 +1029,22 @@ public class DFSInputStream extends FSInputStream
DFSClientFaultInjector.get().fetchFromDatanodeException();
reader = getBlockReader(block, startInBlk, len, datanode.addr,
datanode.storageType, datanode.info);
- int nread = reader.readAll(buf, offset, len);
+
+ //Behave exactly as the readAll() call
+ ByteBuffer tmp = buf.duplicate();
+ tmp.limit(tmp.position() + len);
+ tmp = tmp.slice();
+ int nread = 0;
+ int ret;
+ while (true) {
+ ret = reader.read(tmp);
+ if (ret <= 0) {
+ break;
+ }
+ nread += ret;
+ }
+ buf.position(buf.position() + nread);
+
IOUtilsClient.updateReadStatistics(readStatistics, nread, reader);
dfsClient.updateFileSystemReadStats(
reader.getNetworkDistance(), nread);
@@ -1098,7 +1111,7 @@ public class DFSInputStream extends FSInputStream
* time. We then wait on which ever read returns first.
*/
private void hedgedFetchBlockByteRange(LocatedBlock block, long start,
- long end, byte[] buf, int offset, CorruptedBlocks corruptedBlocks)
+ long end, ByteBuffer buf, CorruptedBlocks corruptedBlocks)
throws IOException {
final DfsClientConf conf = dfsClient.getConf();
ArrayList<Future<ByteBuffer>> futures = new ArrayList<>();
@@ -1130,8 +1143,8 @@ public class DFSInputStream extends FSInputStream
conf.getHedgedReadThresholdMillis(), TimeUnit.MILLISECONDS);
if (future != null) {
ByteBuffer result = future.get();
- System.arraycopy(result.array(), result.position(), buf, offset,
- len);
+ result.flip();
+ buf.put(result);
return;
}
DFSClient.LOG.debug("Waited {}ms to read from {}; spawning hedged "
@@ -1173,8 +1186,8 @@ public class DFSInputStream extends FSInputStream
// cancel the rest.
cancelAll(futures);
dfsClient.getHedgedReadMetrics().incHedgedReadWins();
- System.arraycopy(result.array(), result.position(), buf, offset,
- len);
+ result.flip();
+ buf.put(result);
return;
} catch (InterruptedException ie) {
// Ignore and retry
@@ -1244,7 +1257,8 @@ public class DFSInputStream extends FSInputStream
* access key from its memory since it's considered expired based on
* the estimated expiration date.
*/
- if (ex instanceof InvalidBlockTokenException || ex instanceof InvalidToken) {
+ if (ex instanceof InvalidBlockTokenException ||
+ ex instanceof InvalidToken) {
DFSClient.LOG.info("Access token was invalid when connecting to "
+ targetAddr + " : " + ex);
return true;
@@ -1272,7 +1286,8 @@ public class DFSInputStream extends FSInputStream
try (TraceScope scope = dfsClient.
newReaderTraceScope("DFSInputStream#byteArrayPread",
src, position, length)) {
- int retLen = pread(position, buffer, offset, length);
+ ByteBuffer bb = ByteBuffer.wrap(buffer, offset, length);
+ int retLen = pread(position, bb);
if (retLen < length) {
dfsClient.addRetLenToReaderScope(scope, retLen);
}
@@ -1280,7 +1295,7 @@ public class DFSInputStream extends FSInputStream
}
}
- private int pread(long position, byte[] buffer, int offset, int length)
+ private int pread(long position, ByteBuffer buffer)
throws IOException {
// sanity checks
dfsClient.checkOpen();
@@ -1292,6 +1307,7 @@ public class DFSInputStream extends FSInputStream
if ((position < 0) || (position >= filelen)) {
return -1;
}
+ int length = buffer.remaining();
int realLen = length;
if ((position + length) > filelen) {
realLen = (int)(filelen - position);
@@ -1304,14 +1320,16 @@ public class DFSInputStream extends FSInputStream
CorruptedBlocks corruptedBlocks = new CorruptedBlocks();
for (LocatedBlock blk : blockRange) {
long targetStart = position - blk.getStartOffset();
- long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
+ int bytesToRead = (int) Math.min(remaining,
+ blk.getBlockSize() - targetStart);
+ long targetEnd = targetStart + bytesToRead - 1;
try {
if (dfsClient.isHedgedReadsEnabled() && !blk.isStriped()) {
hedgedFetchBlockByteRange(blk, targetStart,
- targetStart + bytesToRead - 1, buffer, offset, corruptedBlocks);
+ targetEnd, buffer, corruptedBlocks);
} else {
- fetchBlockByteRange(blk, targetStart, targetStart + bytesToRead - 1,
- buffer, offset, corruptedBlocks);
+ fetchBlockByteRange(blk, targetStart, targetEnd,
+ buffer, corruptedBlocks);
}
} finally {
// Check and report if any block replicas are corrupted.
@@ -1323,7 +1341,6 @@ public class DFSInputStream extends FSInputStream
remaining -= bytesToRead;
position += bytesToRead;
- offset += bytesToRead;
}
assert remaining == 0 : "Wrong number of bytes read.";
return realLen;
@@ -1457,7 +1474,8 @@ public class DFSInputStream extends FSInputStream
* If another node could not be found, then returns false.
*/
@Override
- public synchronized boolean seekToNewSource(long targetPos) throws IOException {
+ public synchronized boolean seekToNewSource(long targetPos)
+ throws IOException {
if (currentNode == null) {
return seekToBlockSource(targetPos);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401db4fc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 9ca8005..ccaf6a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -307,8 +307,8 @@ public class DFSStripedInputStream extends DFSInputStream {
stripeLimit - stripeBufOffset);
LocatedStripedBlock blockGroup = (LocatedStripedBlock) currentLocatedBlock;
- AlignedStripe[] stripes = StripedBlockUtil.divideOneStripe(ecPolicy, cellSize,
- blockGroup, offsetInBlockGroup,
+ AlignedStripe[] stripes = StripedBlockUtil.divideOneStripe(ecPolicy,
+ cellSize, blockGroup, offsetInBlockGroup,
offsetInBlockGroup + stripeRange.length - 1, curStripeBuf);
final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(
blockGroup, cellSize, dataBlkNum, parityBlkNum);
@@ -523,13 +523,13 @@ public class DFSStripedInputStream extends DFSInputStream {
*/
@Override
protected void fetchBlockByteRange(LocatedBlock block, long start,
- long end, byte[] buf, int offset, CorruptedBlocks corruptedBlocks)
+ long end, ByteBuffer buf, CorruptedBlocks corruptedBlocks)
throws IOException {
// Refresh the striped block group
LocatedStripedBlock blockGroup = getBlockGroupAt(block.getStartOffset());
AlignedStripe[] stripes = StripedBlockUtil.divideByteRangeIntoStripes(
- ecPolicy, cellSize, blockGroup, start, end, buf, offset);
+ ecPolicy, cellSize, blockGroup, start, end, buf);
CompletionService<Void> readService = new ExecutorCompletionService<>(
dfsClient.getStripedReadsThreadPool());
final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(
@@ -542,6 +542,7 @@ public class DFSStripedInputStream extends DFSInputStream {
blks, preaderInfos, corruptedBlocks);
preader.readStripe();
}
+ buf.position(buf.position() + (int)(end - start + 1));
} finally {
for (BlockReaderInfo preaderInfo : preaderInfos) {
closeReader(preaderInfo);
@@ -698,16 +699,15 @@ public class DFSStripedInputStream extends DFSInputStream {
}
private ByteBufferStrategy[] getReadStrategies(StripingChunk chunk) {
- if (chunk.byteBuffer != null) {
- ByteBufferStrategy strategy =
- new ByteBufferStrategy(chunk.byteBuffer, readStatistics, dfsClient);
+ if (chunk.useByteBuffer()) {
+ ByteBufferStrategy strategy = new ByteBufferStrategy(
+ chunk.getByteBuffer(), readStatistics, dfsClient);
return new ByteBufferStrategy[]{strategy};
} else {
ByteBufferStrategy[] strategies =
- new ByteBufferStrategy[chunk.byteArray.getOffsets().length];
+ new ByteBufferStrategy[chunk.getChunkBuffer().getSlices().size()];
for (int i = 0; i < strategies.length; i++) {
- ByteBuffer buffer = ByteBuffer.wrap(chunk.byteArray.buf(),
- chunk.byteArray.getOffsets()[i], chunk.byteArray.getLengths()[i]);
+ ByteBuffer buffer = chunk.getChunkBuffer().getSlice(i);
strategies[i] =
new ByteBufferStrategy(buffer, readStatistics, dfsClient);
}
@@ -814,7 +814,7 @@ public class DFSStripedInputStream extends DFSInputStream {
}
class PositionStripeReader extends StripeReader {
- private byte[][] decodeInputs = null;
+ private ByteBuffer[] decodeInputs = null;
PositionStripeReader(CompletionService<Void> service,
AlignedStripe alignedStripe, LocatedBlock[] targetBlocks,
@@ -836,8 +836,6 @@ public class DFSStripedInputStream extends DFSInputStream {
Preconditions.checkState(index >= dataBlkNum &&
alignedStripe.chunks[index] == null);
alignedStripe.chunks[index] = new StripingChunk(decodeInputs[index]);
- alignedStripe.chunks[index].addByteArraySlice(0,
- (int) alignedStripe.getSpanInBlock());
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401db4fc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index c8827d9..4dbbc3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -73,7 +73,8 @@ import java.util.concurrent.TimeUnit;
@InterfaceAudience.Private
public class StripedBlockUtil {
- public static final Logger LOG = LoggerFactory.getLogger(StripedBlockUtil.class);
+ public static final Logger LOG =
+ LoggerFactory.getLogger(StripedBlockUtil.class);
/**
* Parses a striped block group into individual blocks.
@@ -312,16 +313,17 @@ public class StripedBlockUtil {
* schedule a new fetch request with the decoding input buffer as transfer
* destination.
*/
- public static byte[][] initDecodeInputs(AlignedStripe alignedStripe,
+ public static ByteBuffer[] initDecodeInputs(AlignedStripe alignedStripe,
int dataBlkNum, int parityBlkNum) {
- byte[][] decodeInputs =
- new byte[dataBlkNum + parityBlkNum][(int) alignedStripe.getSpanInBlock()];
+ ByteBuffer[] decodeInputs = new ByteBuffer[dataBlkNum + parityBlkNum];
+ for (int i = 0; i < decodeInputs.length; i++) {
+ decodeInputs[i] = ByteBuffer.allocate(
+ (int) alignedStripe.getSpanInBlock());
+ }
// read the full data aligned stripe
for (int i = 0; i < dataBlkNum; i++) {
if (alignedStripe.chunks[i] == null) {
alignedStripe.chunks[i] = new StripingChunk(decodeInputs[i]);
- alignedStripe.chunks[i].addByteArraySlice(0,
- (int) alignedStripe.getSpanInBlock());
}
}
return decodeInputs;
@@ -334,14 +336,21 @@ public class StripedBlockUtil {
* When all pending requests have returned, this method should be called to
* finalize decode input buffers.
*/
- public static void finalizeDecodeInputs(final byte[][] decodeInputs,
+ public static void finalizeDecodeInputs(final ByteBuffer[] decodeInputs,
AlignedStripe alignedStripe) {
for (int i = 0; i < alignedStripe.chunks.length; i++) {
final StripingChunk chunk = alignedStripe.chunks[i];
if (chunk != null && chunk.state == StripingChunk.FETCHED) {
- chunk.copyTo(decodeInputs[i]);
+ if (chunk.useChunkBuffer()) {
+ chunk.getChunkBuffer().copyTo(decodeInputs[i]);
+ } else {
+ chunk.getByteBuffer().flip();
+ }
} else if (chunk != null && chunk.state == StripingChunk.ALLZERO) {
- Arrays.fill(decodeInputs[i], (byte) 0);
+ //ZERO it. Will be better handled in other following issue.
+ byte[] emptyBytes = new byte[decodeInputs[i].limit()];
+ decodeInputs[i].put(emptyBytes);
+ decodeInputs[i].flip();
} else {
decodeInputs[i] = null;
}
@@ -351,7 +360,7 @@ public class StripedBlockUtil {
/**
* Decode based on the given input buffers and erasure coding policy.
*/
- public static void decodeAndFillBuffer(final byte[][] decodeInputs,
+ public static void decodeAndFillBuffer(final ByteBuffer[] decodeInputs,
AlignedStripe alignedStripe, int dataBlkNum, int parityBlkNum,
RawErasureDecoder decoder) {
// Step 1: prepare indices and output buffers for missing data units
@@ -364,8 +373,11 @@ public class StripedBlockUtil {
}
}
decodeIndices = Arrays.copyOf(decodeIndices, pos);
- byte[][] decodeOutputs =
- new byte[decodeIndices.length][(int) alignedStripe.getSpanInBlock()];
+ ByteBuffer[] decodeOutputs = new ByteBuffer[decodeIndices.length];
+ for (int i = 0; i < decodeOutputs.length; i++) {
+ decodeOutputs[i] = ByteBuffer.allocate(
+ (int) alignedStripe.getSpanInBlock());
+ }
// Step 2: decode into prepared output buffers
decoder.decode(decodeInputs, decodeIndices, decodeOutputs);
@@ -374,8 +386,8 @@ public class StripedBlockUtil {
for (int i = 0; i < decodeIndices.length; i++) {
int missingBlkIdx = decodeIndices[i];
StripingChunk chunk = alignedStripe.chunks[missingBlkIdx];
- if (chunk.state == StripingChunk.MISSING) {
- chunk.copyFrom(decodeOutputs[i]);
+ if (chunk.state == StripingChunk.MISSING && chunk.useChunkBuffer()) {
+ chunk.getChunkBuffer().copyFrom(decodeOutputs[i]);
}
}
}
@@ -402,7 +414,8 @@ public class StripedBlockUtil {
// Step 4: calculate each chunk's position in destination buffer. Since the
// whole read range is within a single stripe, the logic is simpler here.
- int bufOffset = (int) (rangeStartInBlockGroup % ((long) cellSize * dataBlkNum));
+ int bufOffset =
+ (int) (rangeStartInBlockGroup % ((long) cellSize * dataBlkNum));
for (StripingCell cell : cells) {
long cellStart = cell.idxInInternalBlk * cellSize + cell.offset;
long cellEnd = cellStart + cell.size - 1;
@@ -437,15 +450,14 @@ public class StripedBlockUtil {
* @param rangeStartInBlockGroup The byte range's start offset in block group
* @param rangeEndInBlockGroup The byte range's end offset in block group
* @param buf Destination buffer of the read operation for the byte range
- * @param offsetInBuf Start offset into the destination buffer
*
* At most 5 stripes will be generated from each logical range, as
* demonstrated in the header of {@link AlignedStripe}.
*/
- public static AlignedStripe[] divideByteRangeIntoStripes(ErasureCodingPolicy ecPolicy,
+ public static AlignedStripe[] divideByteRangeIntoStripes(
+ ErasureCodingPolicy ecPolicy,
int cellSize, LocatedStripedBlock blockGroup,
- long rangeStartInBlockGroup, long rangeEndInBlockGroup, byte[] buf,
- int offsetInBuf) {
+ long rangeStartInBlockGroup, long rangeEndInBlockGroup, ByteBuffer buf) {
// Step 0: analyze range and calculate basic parameters
final int dataBlkNum = ecPolicy.getNumDataUnits();
@@ -462,7 +474,7 @@ public class StripedBlockUtil {
AlignedStripe[] stripes = mergeRangesForInternalBlocks(ecPolicy, ranges);
// Step 4: calculate each chunk's position in destination buffer
- calcualteChunkPositionsInBuf(cellSize, stripes, cells, buf, offsetInBuf);
+ calcualteChunkPositionsInBuf(cellSize, stripes, cells, buf);
// Step 5: prepare ALLZERO blocks
prepareAllZeroChunks(blockGroup, stripes, cellSize, dataBlkNum);
@@ -476,7 +488,8 @@ public class StripedBlockUtil {
* used by {@link DFSStripedOutputStream} in encoding
*/
@VisibleForTesting
- private static StripingCell[] getStripingCellsOfByteRange(ErasureCodingPolicy ecPolicy,
+ private static StripingCell[] getStripingCellsOfByteRange(
+ ErasureCodingPolicy ecPolicy,
int cellSize, LocatedStripedBlock blockGroup,
long rangeStartInBlockGroup, long rangeEndInBlockGroup) {
Preconditions.checkArgument(
@@ -511,7 +524,8 @@ public class StripedBlockUtil {
* the physical byte range (inclusive) on each stored internal block.
*/
@VisibleForTesting
- private static VerticalRange[] getRangesForInternalBlocks(ErasureCodingPolicy ecPolicy,
+ private static VerticalRange[] getRangesForInternalBlocks(
+ ErasureCodingPolicy ecPolicy,
int cellSize, StripingCell[] cells) {
int dataBlkNum = ecPolicy.getNumDataUnits();
int parityBlkNum = ecPolicy.getNumParityUnits();
@@ -575,8 +589,7 @@ public class StripedBlockUtil {
}
private static void calcualteChunkPositionsInBuf(int cellSize,
- AlignedStripe[] stripes, StripingCell[] cells, byte[] buf,
- int offsetInBuf) {
+ AlignedStripe[] stripes, StripingCell[] cells, ByteBuffer buf) {
/**
* | <--------------- AlignedStripe --------------->|
*
@@ -598,6 +611,7 @@ public class StripedBlockUtil {
for (StripingCell cell : cells) {
long cellStart = cell.idxInInternalBlk * cellSize + cell.offset;
long cellEnd = cellStart + cell.size - 1;
+ StripingChunk chunk;
for (AlignedStripe s : stripes) {
long stripeEnd = s.getOffsetInBlock() + s.getSpanInBlock() - 1;
long overlapStart = Math.max(cellStart, s.getOffsetInBlock());
@@ -606,11 +620,13 @@ public class StripedBlockUtil {
if (overLapLen <= 0) {
continue;
}
- if (s.chunks[cell.idxInStripe] == null) {
- s.chunks[cell.idxInStripe] = new StripingChunk(buf);
+ chunk = s.chunks[cell.idxInStripe];
+ if (chunk == null) {
+ chunk = new StripingChunk();
+ s.chunks[cell.idxInStripe] = chunk;
}
- s.chunks[cell.idxInStripe].addByteArraySlice(
- (int)(offsetInBuf + done + overlapStart - cellStart), overLapLen);
+ chunk.getChunkBuffer().addSlice(buf,
+ (int) (done + overlapStart - cellStart), overLapLen);
}
done += cell.size;
}
@@ -833,88 +849,89 @@ public class StripedBlockUtil {
*/
public int state = REQUESTED;
- public final ChunkByteArray byteArray;
- public final ByteBuffer byteBuffer;
+ private final ChunkByteBuffer chunkBuffer;
+ private final ByteBuffer byteBuffer;
- public StripingChunk(byte[] buf) {
- this.byteArray = new ChunkByteArray(buf);
+ public StripingChunk() {
+ this.chunkBuffer = new ChunkByteBuffer();
byteBuffer = null;
}
public StripingChunk(ByteBuffer buf) {
- this.byteArray = null;
+ this.chunkBuffer = null;
this.byteBuffer = buf;
}
public StripingChunk(int state) {
- this.byteArray = null;
+ this.chunkBuffer = null;
this.byteBuffer = null;
this.state = state;
}
- public void addByteArraySlice(int offset, int length) {
- assert byteArray != null;
- byteArray.offsetsInBuf.add(offset);
- byteArray.lengthsInBuf.add(length);
+ public boolean useByteBuffer(){
+ return byteBuffer != null;
}
- void copyTo(byte[] target) {
- assert byteArray != null;
- byteArray.copyTo(target);
+ public boolean useChunkBuffer() {
+ return chunkBuffer != null;
}
- void copyFrom(byte[] src) {
- assert byteArray != null;
- byteArray.copyFrom(src);
+ public ByteBuffer getByteBuffer() {
+ assert byteBuffer != null;
+ return byteBuffer;
+ }
+
+ public ChunkByteBuffer getChunkBuffer() {
+ assert chunkBuffer != null;
+ return chunkBuffer;
}
}
- public static class ChunkByteArray {
- private final byte[] buf;
- private final List<Integer> offsetsInBuf;
- private final List<Integer> lengthsInBuf;
+ /**
+ * A utility to manage ByteBuffer slices for a reader.
+ */
+ public static class ChunkByteBuffer {
+ private final List<ByteBuffer> slices;
- ChunkByteArray(byte[] buf) {
- this.buf = buf;
- this.offsetsInBuf = new ArrayList<>();
- this.lengthsInBuf = new ArrayList<>();
+ ChunkByteBuffer() {
+ this.slices = new ArrayList<>();
}
- public int[] getOffsets() {
- int[] offsets = new int[offsetsInBuf.size()];
- for (int i = 0; i < offsets.length; i++) {
- offsets[i] = offsetsInBuf.get(i);
- }
- return offsets;
+ public void addSlice(ByteBuffer buffer, int offset, int len) {
+ ByteBuffer tmp = buffer.duplicate();
+ tmp.position(buffer.position() + offset);
+ tmp.limit(buffer.position() + offset + len);
+ slices.add(tmp.slice());
}
- public int[] getLengths() {
- int[] lens = new int[this.lengthsInBuf.size()];
- for (int i = 0; i < lens.length; i++) {
- lens[i] = this.lengthsInBuf.get(i);
- }
- return lens;
+ public ByteBuffer getSlice(int i) {
+ return slices.get(i);
}
- public byte[] buf() {
- return buf;
+ public List<ByteBuffer> getSlices() {
+ return slices;
}
- void copyTo(byte[] target) {
- int posInBuf = 0;
- for (int i = 0; i < offsetsInBuf.size(); i++) {
- System.arraycopy(buf, offsetsInBuf.get(i),
- target, posInBuf, lengthsInBuf.get(i));
- posInBuf += lengthsInBuf.get(i);
+ /**
+ * Note: target will be ready-to-read state after the call.
+ */
+ void copyTo(ByteBuffer target) {
+ for (ByteBuffer slice : slices) {
+ slice.flip();
+ target.put(slice);
}
- }
-
- void copyFrom(byte[] src) {
- int srcPos = 0;
- for (int j = 0; j < offsetsInBuf.size(); j++) {
- System.arraycopy(src, srcPos, buf, offsetsInBuf.get(j),
- lengthsInBuf.get(j));
- srcPos += lengthsInBuf.get(j);
+ target.flip();
+ }
+
+ void copyFrom(ByteBuffer src) {
+ ByteBuffer tmp;
+ int len;
+ for (ByteBuffer slice : slices) {
+ len = slice.remaining();
+ tmp = src.duplicate();
+ tmp.limit(tmp.position() + len);
+ slice.put(tmp);
+ src.position(src.position() + len);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401db4fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 18c2de9..1e27745 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -57,7 +57,8 @@ import static org.junit.Assert.assertTrue;
public class TestDFSStripedInputStream {
- public static final Log LOG = LogFactory.getLog(TestDFSStripedInputStream.class);
+ public static final Log LOG =
+ LogFactory.getLog(TestDFSStripedInputStream.class);
private MiniDFSCluster cluster;
private Configuration conf = new Configuration();
@@ -272,12 +273,16 @@ public class TestDFSStripedInputStream {
// |10 |
done += in.read(0, readBuffer, 0, delta);
assertEquals(delta, done);
+ assertArrayEquals(Arrays.copyOf(expected, done),
+ Arrays.copyOf(readBuffer, done));
// both head and trail cells are partial
// |c_0 |c_1 |c_2 |c_3 |c_4 |c_5 |
// |256K - 10|missing|256K|256K|256K - 10|not in range|
done += in.read(delta, readBuffer, delta,
CELLSIZE * (DATA_BLK_NUM - 1) - 2 * delta);
assertEquals(CELLSIZE * (DATA_BLK_NUM - 1) - delta, done);
+ assertArrayEquals(Arrays.copyOf(expected, done),
+ Arrays.copyOf(readBuffer, done));
// read the rest
done += in.read(done, readBuffer, done, readSize - done);
assertEquals(readSize, done);
@@ -291,8 +296,8 @@ public class TestDFSStripedInputStream {
testStatefulRead(true, true);
}
- private void testStatefulRead(boolean useByteBuffer, boolean cellMisalignPacket)
- throws Exception {
+ private void testStatefulRead(boolean useByteBuffer,
+ boolean cellMisalignPacket) throws Exception {
final int numBlocks = 2;
final int fileSize = numBlocks * BLOCK_GROUP_SIZE;
if (cellMisalignPacket) {
@@ -302,7 +307,8 @@ public class TestDFSStripedInputStream {
}
DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
NUM_STRIPE_PER_BLOCK, false);
- LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(filePath.toString(), 0, fileSize);
+ LocatedBlocks lbs = fs.getClient().namenode.
+ getBlockLocations(filePath.toString(), 0, fileSize);
assert lbs.getLocatedBlocks().size() == numBlocks;
for (LocatedBlock lb : lbs.getLocatedBlocks()) {
@@ -360,4 +366,111 @@ public class TestDFSStripedInputStream {
}
fs.delete(filePath, true);
}
+
+ @Test
+ public void testStatefulReadWithDNFailure() throws Exception {
+ final int numBlocks = 4;
+ final int failedDNIdx = DATA_BLK_NUM - 1;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
+ filePath.toString(), 0, BLOCK_GROUP_SIZE);
+
+ assert lbs.get(0) instanceof LocatedStripedBlock;
+ LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
+ for (int i = 0; i < DATA_BLK_NUM + PARITY_BLK_NUM; i++) {
+ Block blk = new Block(bg.getBlock().getBlockId() + i,
+ NUM_STRIPE_PER_BLOCK * CELLSIZE,
+ bg.getBlock().getGenerationStamp());
+ blk.setGenerationStamp(bg.getBlock().getGenerationStamp());
+ cluster.injectBlocks(i, Arrays.asList(blk),
+ bg.getBlock().getBlockPoolId());
+ }
+ DFSStripedInputStream in =
+ new DFSStripedInputStream(fs.getClient(), filePath.toString(), false,
+ ecPolicy, null);
+ int readSize = BLOCK_GROUP_SIZE;
+ byte[] readBuffer = new byte[readSize];
+ byte[] expected = new byte[readSize];
+ /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
+ for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ for (int k = 0; k < CELLSIZE; k++) {
+ int posInBlk = i * CELLSIZE + k;
+ int posInFile = i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE + k;
+ expected[posInFile] = SimulatedFSDataset.simulatedByte(
+ new Block(bg.getBlock().getBlockId() + j), posInBlk);
+ }
+ }
+ }
+
+ ErasureCoderOptions coderOptions = new ErasureCoderOptions(
+ DATA_BLK_NUM, PARITY_BLK_NUM);
+ RawErasureDecoder rawDecoder = CodecUtil.createRawDecoder(conf,
+ ecPolicy.getCodecName(), coderOptions);
+
+ // Update the expected content for decoded data
+ int[] missingBlkIdx = new int[PARITY_BLK_NUM];
+ for (int i = 0; i < missingBlkIdx.length; i++) {
+ if (i == 0) {
+ missingBlkIdx[i] = failedDNIdx;
+ } else {
+ missingBlkIdx[i] = DATA_BLK_NUM + i;
+ }
+ }
+ cluster.stopDataNode(failedDNIdx);
+ for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
+ byte[][] decodeInputs = new byte[DATA_BLK_NUM + PARITY_BLK_NUM][CELLSIZE];
+ byte[][] decodeOutputs = new byte[missingBlkIdx.length][CELLSIZE];
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ int posInBuf = i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE;
+ if (j != failedDNIdx) {
+ System.arraycopy(expected, posInBuf, decodeInputs[j], 0, CELLSIZE);
+ }
+ }
+ for (int j = DATA_BLK_NUM; j < DATA_BLK_NUM + PARITY_BLK_NUM; j++) {
+ for (int k = 0; k < CELLSIZE; k++) {
+ int posInBlk = i * CELLSIZE + k;
+ decodeInputs[j][k] = SimulatedFSDataset.simulatedByte(
+ new Block(bg.getBlock().getBlockId() + j), posInBlk);
+ }
+ }
+ for (int m : missingBlkIdx) {
+ decodeInputs[m] = null;
+ }
+ rawDecoder.decode(decodeInputs, missingBlkIdx, decodeOutputs);
+ int posInBuf = i * CELLSIZE * DATA_BLK_NUM + failedDNIdx * CELLSIZE;
+ System.arraycopy(decodeOutputs[0], 0, expected, posInBuf, CELLSIZE);
+ }
+
+ int delta = 10;
+ int done = 0;
+ // read a small delta, shouldn't trigger decode
+ // |cell_0 |
+ // |10 |
+ done += in.read(readBuffer, 0, delta);
+ assertEquals(delta, done);
+ // both head and trail cells are partial
+ // |c_0 |c_1 |c_2 |c_3 |c_4 |c_5 |
+ // |256K - 10|missing|256K|256K|256K - 10|not in range|
+ while (done < (CELLSIZE * (DATA_BLK_NUM - 1) - 2 * delta)) {
+ int ret = in.read(readBuffer, delta,
+ CELLSIZE * (DATA_BLK_NUM - 1) - 2 * delta);
+ assertTrue(ret > 0);
+ done += ret;
+ }
+ assertEquals(CELLSIZE * (DATA_BLK_NUM - 1) - delta, done);
+ // read the rest
+
+ int restSize;
+ restSize = readSize - done;
+ while (done < restSize) {
+ int ret = in.read(readBuffer, done, restSize);
+ assertTrue(ret > 0);
+ done += ret;
+ }
+
+ assertEquals(readSize, done);
+ assertArrayEquals(expected, readBuffer);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401db4fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
index 96fc79c..7d9d7dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
@@ -36,6 +36,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import java.nio.ByteBuffer;
import java.util.Random;
import static org.junit.Assert.assertEquals;
@@ -242,7 +243,8 @@ public class TestStripedBlockUtil {
*/
@Test
public void testDivideByteRangeIntoStripes() {
- byte[] assembled = new byte[BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE];
+ ByteBuffer assembled =
+ ByteBuffer.allocate(BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE);
for (int bgSize : blockGroupSizes) {
LocatedStripedBlock blockGroup = createDummyLocatedBlock(bgSize);
byte[][] internalBlkBufs = createInternalBlkBuffers(bgSize);
@@ -252,7 +254,7 @@ public class TestStripedBlockUtil {
continue;
}
AlignedStripe[] stripes = divideByteRangeIntoStripes(EC_POLICY,
- CELLSIZE, blockGroup, brStart, brStart + brSize - 1, assembled, 0);
+ CELLSIZE, blockGroup, brStart, brStart + brSize - 1, assembled);
for (AlignedStripe stripe : stripes) {
for (int i = 0; i < DATA_BLK_NUM; i++) {
@@ -261,21 +263,21 @@ public class TestStripedBlockUtil {
continue;
}
int done = 0;
- for (int j = 0; j < chunk.byteArray.getLengths().length; j++) {
- System.arraycopy(internalBlkBufs[i],
- (int) stripe.getOffsetInBlock() + done, assembled,
- chunk.byteArray.getOffsets()[j],
- chunk.byteArray.getLengths()[j]);
- done += chunk.byteArray.getLengths()[j];
+ int len;
+ for (ByteBuffer slice : chunk.getChunkBuffer().getSlices()) {
+ len = slice.remaining();
+ slice.put(internalBlkBufs[i],
+ (int) stripe.getOffsetInBlock() + done, len);
+ done += len;
}
}
}
for (int i = 0; i < brSize; i++) {
- if (hashIntToByte(brStart + i) != assembled[i]) {
+ if (hashIntToByte(brStart + i) != assembled.get(i)) {
System.out.println("Oops");
}
assertEquals("Byte at " + (brStart + i) + " should be the same",
- hashIntToByte(brStart + i), assembled[i]);
+ hashIntToByte(brStart + i), assembled.get(i));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/43] hadoop git commit: YARN-5566. Client-side NM graceful decom
is not triggered when jobs finish. (Robert Kanter via kasha)
Posted by ae...@apache.org.
YARN-5566. Client-side NM graceful decom is not triggered when jobs finish. (Robert Kanter via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74f4bae4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74f4bae4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74f4bae4
Branch: refs/heads/HDFS-7240
Commit: 74f4bae45597f4794e99e33309130ddff647b21f
Parents: 76cd81f
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Thu Sep 1 14:44:01 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Thu Sep 1 14:44:01 2016 -0700
----------------------------------------------------------------------
.../resourcemanager/rmnode/RMNodeImpl.java | 13 +++++++--
.../resourcemanager/TestRMNodeTransitions.java | 29 ++++++++++++++------
.../TestResourceTrackerService.java | 4 ++-
3 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74f4bae4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index d1ccecb..375b4cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -1170,12 +1170,21 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
NodeState initialState = rmNode.getState();
boolean isNodeDecommissioning =
initialState.equals(NodeState.DECOMMISSIONING);
+ if (isNodeDecommissioning) {
+ List<ApplicationId> keepAliveApps = statusEvent.getKeepAliveAppIds();
+ if (rmNode.runningApplications.isEmpty() &&
+ (keepAliveApps == null || keepAliveApps.isEmpty())) {
+ RMNodeImpl.deactivateNode(rmNode, NodeState.DECOMMISSIONED);
+ return NodeState.DECOMMISSIONED;
+ }
+ }
+
if (!remoteNodeHealthStatus.getIsNodeHealthy()) {
LOG.info("Node " + rmNode.nodeId +
" reported UNHEALTHY with details: " +
remoteNodeHealthStatus.getHealthReport());
// if a node in decommissioning receives an unhealthy report,
- // it will keep decommissioning.
+ // it will stay in decommissioning.
if (isNodeDecommissioning) {
return NodeState.DECOMMISSIONING;
} else {
@@ -1349,7 +1358,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
+ " is the first container get launched for application "
+ containerAppId);
}
- runningApplications.add(containerAppId);
+ handleRunningAppOnNode(this, context, containerAppId, nodeId);
}
// Process running containers
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74f4bae4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index e82b93c..6038b31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer
.AllocationExpirationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -73,6 +74,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -279,16 +281,17 @@ public class TestRMNodeTransitions {
NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
node2.handle(new RMNodeStartedEvent(null, null, null));
-
+
+ ApplicationId app0 = BuilderUtils.newApplicationId(0, 0);
+ ApplicationId app1 = BuilderUtils.newApplicationId(1, 1);
ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(0, 0), 0), 0);
+ BuilderUtils.newApplicationAttemptId(app0, 0), 0);
ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(1, 1), 1), 1);
+ BuilderUtils.newApplicationAttemptId(app1, 1), 1);
ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(1, 1), 1), 2);
+ BuilderUtils.newApplicationAttemptId(app1, 1), 2);
+ rmContext.getRMApps().put(app0, Mockito.mock(RMApp.class));
+ rmContext.getRMApps().put(app1, Mockito.mock(RMApp.class));
RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent(null);
RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent(null);
@@ -652,6 +655,7 @@ public class TestRMNodeTransitions {
NodeId nodeId = node.getNodeID();
ApplicationId runningAppId = BuilderUtils.newApplicationId(0, 1);
+ rmContext.getRMApps().put(runningAppId, Mockito.mock(RMApp.class));
// Create a running container
ContainerId runningContainerId = BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(
@@ -919,16 +923,22 @@ public class TestRMNodeTransitions {
}
// Test unhealthy report on a decommissioning node will make it
- // keep decommissioning.
+ // keep decommissioning as long as there's a running or keep alive app.
+ // Otherwise, it will go to decommissioned
@Test
public void testDecommissioningUnhealthy() {
RMNodeImpl node = getDecommissioningNode();
NodeHealthStatus status = NodeHealthStatus.newInstance(false, "sick",
System.currentTimeMillis());
+ List<ApplicationId> keepAliveApps = new ArrayList<>();
+ keepAliveApps.add(BuilderUtils.newApplicationId(1, 1));
NodeStatus nodeStatus = NodeStatus.newInstance(node.getNodeID(), 0,
- new ArrayList<ContainerStatus>(), null, status, null, null, null);
+ null, keepAliveApps, status, null, null, null);
node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null));
Assert.assertEquals(NodeState.DECOMMISSIONING, node.getState());
+ nodeStatus.setKeepAliveApplications(null);
+ node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null));
+ Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}
@Test
@@ -951,6 +961,7 @@ public class TestRMNodeTransitions {
ApplicationId.newInstance(System.currentTimeMillis(), 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
+ rmContext.getRMApps().put(appId, Mockito.mock(RMApp.class));
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1L);
ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2L);
AllocationExpirationInfo expirationInfo1 =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74f4bae4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index c2a20a1..916c942 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -299,6 +299,8 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
RMApp app = rm.submitApp(2000);
MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
ApplicationAttemptId aaid = app.getCurrentAppAttempt().getAppAttemptId();
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+ nm3.nodeHeartbeat(true);
// Graceful decommission host1 and host3
writeToHostsFile("host1", "host3");
@@ -308,7 +310,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
// host1 should be DECOMMISSIONING due to running containers.
// host3 should become DECOMMISSIONED.
- nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+ nm1.nodeHeartbeat(true);
nm3.nodeHeartbeat(true);
rm.waitForState(id1, NodeState.DECOMMISSIONING);
rm.waitForState(id3, NodeState.DECOMMISSIONED);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/43] hadoop git commit: HDFS-10820. Reuse closeResponder to reset
the response variable in DataStreamer#run. Contributed by Yiqun Lin.
Posted by ae...@apache.org.
HDFS-10820. Reuse closeResponder to reset the response variable in DataStreamer#run. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0690f096
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0690f096
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0690f096
Branch: refs/heads/HDFS-7240
Commit: 0690f0969efef201b074a8b26400b2e5ba9b97ad
Parents: 74f4bae
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Sep 1 15:34:47 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Sep 1 15:35:25 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DataStreamer.java | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0690f096/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index ee673ed..062a416 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -536,14 +536,8 @@ class DataStreamer extends Daemon {
TraceScope scope = null;
while (!streamerClosed && dfsClient.clientRunning) {
// if the Responder encountered an error, shutdown Responder
- if (errorState.hasError() && response != null) {
- try {
- response.close();
- response.join();
- response = null;
- } catch (InterruptedException e) {
- LOG.warn("Caught exception", e);
- }
+ if (errorState.hasError()) {
+ closeResponder();
}
DFSPacket one;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/43] hadoop git commit: HADOOP-13388. Clean up
TestLocalFileSystemPermission. Contributed by Andras Bokor.
Posted by ae...@apache.org.
HADOOP-13388. Clean up TestLocalFileSystemPermission. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f414d5e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f414d5e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f414d5e1
Branch: refs/heads/HDFS-7240
Commit: f414d5e118940cb98015c0b66e11102a9704a505
Parents: 7fdfcd8
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 7 11:09:41 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 7 11:09:41 2016 -0700
----------------------------------------------------------------------
.../fs/TestLocalFileSystemPermission.java | 149 ++++++++-----------
1 file changed, 60 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f414d5e1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
index 817285c..1478111 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
@@ -18,24 +18,23 @@
package org.apache.hadoop.fs;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.apache.hadoop.util.Shell;
-
-import java.io.*;
-import java.util.*;
-
+import org.apache.log4j.Level;
import org.junit.Test;
-import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.*;
/**
* This class tests the local file system via the FileSystem abstraction.
@@ -60,18 +59,22 @@ public class TestLocalFileSystemPermission {
return f;
}
- private Path writeFile(FileSystem fs, String name, FsPermission perm) throws IOException {
+ private Path writeFile(FileSystem fs, String name, FsPermission perm)
+ throws IOException {
Path f = new Path(TEST_PATH_PREFIX + name);
- FSDataOutputStream stm = fs.create(f, perm, true, 2048, (short)1, 32 * 1024 * 1024, null);
+ FSDataOutputStream stm = fs.create(f, perm, true, 2048, (short)1,
+ 32 * 1024 * 1024, null);
stm.writeBytes("42\n");
stm.close();
return f;
}
private void cleanup(FileSystem fs, Path name) throws IOException {
- assertTrue(fs.exists(name));
- fs.delete(name, true);
- assertTrue(!fs.exists(name));
+ if (name!=null) {
+ assertTrue(fs.exists(name));
+ fs.delete(name, true);
+ assertFalse(fs.exists(name));
+ }
}
@Test
@@ -82,39 +85,33 @@ public class TestLocalFileSystemPermission {
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
Path dir = new Path(TEST_PATH_PREFIX + "dir");
localfs.mkdirs(dir);
+ Path dir1 = new Path(TEST_PATH_PREFIX + "dir1");
+ Path dir2 = new Path(TEST_PATH_PREFIX + "dir2");
+
try {
FsPermission initialPermission = getPermission(localfs, dir);
assertEquals(
- FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(conf)),
+ FsPermission.getDirDefault()
+ .applyUMask(FsPermission.getUMask(conf)),
initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- FsPermission perm = new FsPermission((short)0755);
- Path dir1 = new Path(TEST_PATH_PREFIX + "dir1");
- localfs.mkdirs(dir1, perm);
- try {
- FsPermission initialPermission = getPermission(localfs, dir1);
- assertEquals(perm.applyUMask(FsPermission.getUMask(conf)), initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
+ FsPermission perm = new FsPermission((short)0755);
- Path dir2 = new Path(TEST_PATH_PREFIX + "dir2");
- localfs.mkdirs(dir2);
- try {
- FsPermission initialPermission = getPermission(localfs, dir2);
+ localfs.mkdirs(dir1, perm);
+
+ initialPermission = getPermission(localfs, dir1);
+ assertEquals(perm.applyUMask(FsPermission.getUMask(conf)),
+ initialPermission);
+
+ localfs.mkdirs(dir2);
+
+ initialPermission = getPermission(localfs, dir2);
Path copyPath = new Path(TEST_PATH_PREFIX + "dir_copy");
localfs.rename(dir2, copyPath);
FsPermission copyPermission = getPermission(localfs, copyPath);
- assertEquals(copyPermission, initialPermission);
+ assertEquals(initialPermission, copyPermission);
dir2 = copyPath;
- } catch (Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
+
} finally {
cleanup(localfs, dir);
cleanup(localfs, dir1);
@@ -124,52 +121,42 @@ public class TestLocalFileSystemPermission {
}
}
- /** Test LocalFileSystem.setPermission */
+ /** Test LocalFileSystem.setPermission. */
@Test
public void testLocalFSsetPermission() throws IOException {
assumeNotWindows();
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
LocalFileSystem localfs = FileSystem.getLocal(conf);
+ Path f = null;
+ Path f1 = null;
+ Path f2 = null;
String filename = "foo";
- Path f = writeFile(localfs, filename);
+ String filename1 = "foo1";
+ String filename2 = "foo2";
+ FsPermission perm = new FsPermission((short)0755);
+
try {
+ f = writeFile(localfs, filename);
+ f1 = writeFile(localfs, filename1, perm);
+ f2 = writeFile(localfs, filename2);
+
FsPermission initialPermission = getPermission(localfs, f);
assertEquals(
FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)),
initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- String filename1 = "foo1";
- FsPermission perm = new FsPermission((short)0755);
- Path f1 = writeFile(localfs, filename1, perm);
- try {
- FsPermission initialPermission = getPermission(localfs, f1);
+ initialPermission = getPermission(localfs, f1);
assertEquals(
perm.applyUMask(FsPermission.getUMask(conf)), initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- String filename2 = "foo2";
- Path f2 = writeFile(localfs, filename2);
- try {
- FsPermission initialPermission = getPermission(localfs, f2);
+ initialPermission = getPermission(localfs, f2);
Path copyPath = new Path(TEST_PATH_PREFIX + "/foo_copy");
localfs.rename(f2, copyPath);
FsPermission copyPermission = getPermission(localfs, copyPath);
assertEquals(copyPermission, initialPermission);
f2 = copyPath;
- } catch (Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- try {
// create files and manipulate them.
FsPermission all = new FsPermission((short)0777);
FsPermission none = new FsPermission((short)0);
@@ -179,8 +166,7 @@ public class TestLocalFileSystemPermission {
localfs.setPermission(f, all);
assertEquals(all, getPermission(localfs, f));
- }
- finally {
+ } finally {
cleanup(localfs, f);
cleanup(localfs, f1);
if (localfs.exists(f2)) {
@@ -196,33 +182,19 @@ public class TestLocalFileSystemPermission {
/** Test LocalFileSystem.setOwner. */
@Test
public void testLocalFSsetOwner() throws IOException {
- if (Path.WINDOWS) {
- LOGGER.info("Cannot run test for Windows");
- return;
- }
+ assumeNotWindows();
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
LocalFileSystem localfs = FileSystem.getLocal(conf);
String filename = "bar";
Path f = writeFile(localfs, filename);
- List<String> groups = null;
+ List<String> groups;
try {
groups = getGroups();
LOGGER.info("{}: {}", filename, getPermission(localfs, f));
- }
- catch(IOException e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- if (groups == null || groups.size() < 1) {
- LOGGER.error("Cannot run test: need at least one group. groups={}",
- groups);
- return;
- }
- // create files and manipulate them.
- try {
+ // create files and manipulate them.
String g0 = groups.get(0);
localfs.setOwner(f, null, g0);
assertEquals(g0, getGroup(localfs, f));
@@ -235,8 +207,9 @@ public class TestLocalFileSystemPermission {
LOGGER.info("Not testing changing the group since user " +
"belongs to only one group.");
}
- }
- finally {cleanup(localfs, f);}
+ } finally {
+ cleanup(localfs, f);
+ }
}
/**
@@ -250,10 +223,7 @@ public class TestLocalFileSystemPermission {
*/
@Test
public void testSetUmaskInRealTime() throws Exception {
- if (Path.WINDOWS) {
- LOGGER.info("Cannot run test for Windows");
- return;
- }
+ assumeNotWindows();
LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
Configuration conf = localfs.getConf();
@@ -289,9 +259,10 @@ public class TestLocalFileSystemPermission {
}
static List<String> getGroups() throws IOException {
- List<String> a = new ArrayList<String>();
+ List<String> a = new ArrayList<>();
String s = Shell.execCommand(Shell.getGroupsCommand());
- for(StringTokenizer t = new StringTokenizer(s); t.hasMoreTokens(); ) {
+ StringTokenizer t = new StringTokenizer(s);
+ while (t.hasMoreTokens()) {
a.add(t.nextToken());
}
return a;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/43] hadoop git commit: HDFS-10833. Fix JSON errors in WebHDFS.md
examples.
Posted by ae...@apache.org.
HDFS-10833. Fix JSON errors in WebHDFS.md examples.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbd909ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbd909ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbd909ce
Branch: refs/heads/HDFS-7240
Commit: cbd909ce2a5ac1da258f756fa1f93e84dd20b926
Parents: 378f624
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Sep 2 14:50:34 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Sep 2 14:50:34 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbd909ce/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 30f583e..c62fb2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -1294,7 +1294,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
{
"entries":
{
- "type": "array"
+ "type": "array",
"items":
{
"description": "ACL entry.",
@@ -1318,7 +1318,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
"description": "True if the sticky bit is on.",
"type" : "boolean",
"required" : true
- },
+ }
}
}
}
@@ -1337,7 +1337,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
"type" : "array",
"items":
{
- "type" " "object",
+ "type" : "object",
"properties":
{
"name":
@@ -1368,7 +1368,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
"XAttrNames":
{
"description": "XAttr names.",
- "type" : "string"
+ "type" : "string",
"required" : true
}
}
@@ -1737,7 +1737,7 @@ See also: [`GETHOMEDIRECTORY`](#Get_Home_Directory), [Path](../../api/org/apache
"javaClassName": //an optional property
{
"description": "Java class name of the exception",
- "type" : "string",
+ "type" : "string"
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/43] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f388a209/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
new file mode 100644
index 0000000..e539a80
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
@@ -0,0 +1,3712 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" Changelog
+
+## Release 3.0.0-alpha1 - 2016-08-30
+
+### INCOMPATIBLE CHANGES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-8124](https://issues.apache.org/jira/browse/HADOOP-8124) | Remove the deprecated Syncable.sync() method | Major | fs | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-3034](https://issues.apache.org/jira/browse/HDFS-3034) | Remove the deprecated Syncable.sync() method | Major | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-5079](https://issues.apache.org/jira/browse/HDFS-5079) | Cleaning up NNHAStatusHeartbeat.State DatanodeProtocolProtos. | Major | datanode, namenode | Konstantin Shvachko | Tao Luo |
+| [HDFS-5570](https://issues.apache.org/jira/browse/HDFS-5570) | Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs | Major | . | Haohui Mai | Haohui Mai |
+| [HADOOP-10485](https://issues.apache.org/jira/browse/HADOOP-10485) | Remove dead classes in hadoop-streaming | Major | tools | Haohui Mai | Haohui Mai |
+| [HDFS-6246](https://issues.apache.org/jira/browse/HDFS-6246) | Remove 'dfs.support.append' flag from trunk code | Minor | . | Uma Maheswara Rao G | Uma Maheswara Rao G |
+| [HADOOP-10474](https://issues.apache.org/jira/browse/HADOOP-10474) | Move o.a.h.record to hadoop-streaming | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-2538](https://issues.apache.org/jira/browse/HDFS-2538) | option to disable fsck dots | Minor | . | Allen Wittenauer | Mohammad Kamrul Islam |
+| [HADOOP-9902](https://issues.apache.org/jira/browse/HADOOP-9902) | Shell script rewrite | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11041](https://issues.apache.org/jira/browse/HADOOP-11041) | VersionInfo output specifies subversion | Minor | build | Guo Ruijing | Tsuyoshi Ozawa |
+| [MAPREDUCE-5972](https://issues.apache.org/jira/browse/MAPREDUCE-5972) | Fix typo 'programatically' in job.xml (and a few other places) | Trivial | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-11356](https://issues.apache.org/jira/browse/HADOOP-11356) | Removed deprecated o.a.h.fs.permission.AccessControlException | Major | . | Haohui Mai | Li Lu |
+| [HADOOP-10950](https://issues.apache.org/jira/browse/HADOOP-10950) | rework heap management vars | Major | scripts | Allen Wittenauer | John Smith |
+| [MAPREDUCE-5785](https://issues.apache.org/jira/browse/MAPREDUCE-5785) | Derive heap size or mapreduce.\*.memory.mb automatically | Major | mr-am, task | Gera Shegalov | Gera Shegalov |
+| [YARN-2428](https://issues.apache.org/jira/browse/YARN-2428) | LCE default banned user list should have yarn | Trivial | nodemanager | Allen Wittenauer | Varun Saxena |
+| [MAPREDUCE-5653](https://issues.apache.org/jira/browse/MAPREDUCE-5653) | DistCp does not honour config-overrides for mapreduce.[map,reduce].memory.mb | Major | distcp | Mithun Radhakrishnan | Ratandeep Ratti |
+| [HADOOP-11657](https://issues.apache.org/jira/browse/HADOOP-11657) | Align the output of `hadoop fs -du` to be more Unix-like | Minor | . | Nick Dimiduk | Akira Ajisaka |
+| [HDFS-7302](https://issues.apache.org/jira/browse/HDFS-7302) | namenode -rollingUpgrade downgrade may finalize a rolling upgrade | Major | namenode | Tsz Wo Nicholas Sze | Kai Sasaki |
+| [HADOOP-6857](https://issues.apache.org/jira/browse/HADOOP-6857) | FsShell should report raw disk usage including replication factor | Major | fs | Alex Kozlov | Byron Wong |
+| [HADOOP-10115](https://issues.apache.org/jira/browse/HADOOP-10115) | Exclude duplicate jars in hadoop package under different component's lib | Major | build | Vinayakumar B | Vinayakumar B |
+| [YARN-3154](https://issues.apache.org/jira/browse/YARN-3154) | Should not upload partial logs for MR jobs or other "short-running' applications | Blocker | nodemanager, resourcemanager | Xuan Gong | Xuan Gong |
+| [MAPREDUCE-4424](https://issues.apache.org/jira/browse/MAPREDUCE-4424) | 'mapred job -list' command should show the job name as well | Minor | mrv2 | Harsh J | Avinash Kujur |
+| [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | FairScheduler handles "invalid" queue names inconsistently | Major | fairscheduler | zhihai xu | zhihai xu |
+| [HDFS-7985](https://issues.apache.org/jira/browse/HDFS-7985) | WebHDFS should be always enabled | Major | . | Haohui Mai | Li Lu |
+| [HDFS-6353](https://issues.apache.org/jira/browse/HDFS-6353) | Check and make checkpoint before stopping the NameNode | Major | namenode | Suresh Srinivas | Jing Zhao |
+| [HADOOP-11553](https://issues.apache.org/jira/browse/HADOOP-11553) | Formalize the shell API | Blocker | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | Rework the changelog and releasenotes | Major | documentation | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | rewrite test-patch.sh | Major | build, test | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11627](https://issues.apache.org/jira/browse/HADOOP-11627) | Remove io.native.lib.available | Major | native | Akira Ajisaka | Brahma Reddy Battula |
+| [HDFS-7281](https://issues.apache.org/jira/browse/HDFS-7281) | Missing block is marked as corrupted block | Major | namenode | Ming Ma | Ming Ma |
+| [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | Non-HA rollback compatibility broken | Blocker | . | J.Andreina | J.Andreina |
+| [MAPREDUCE-2632](https://issues.apache.org/jira/browse/MAPREDUCE-2632) | Avoid calling the partitioner when the numReduceTasks is 1. | Major | . | Ravi Teja Ch N V | Sunil G |
+| [HDFS-8241](https://issues.apache.org/jira/browse/HDFS-8241) | Remove unused NameNode startup option -finalize | Minor | namenode | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8332](https://issues.apache.org/jira/browse/HDFS-8332) | DFS client API calls should check filesystem closed | Major | . | Rakesh R | Rakesh R |
+| [HADOOP-11698](https://issues.apache.org/jira/browse/HADOOP-11698) | Remove DistCpV1 and Logalyzer | Major | tools/distcp | Allen Wittenauer | Brahma Reddy Battula |
+| [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | RPC Invoker relies on static ClientCache which has synchronized(this) blocks | Major | ipc, performance | Gopal V | Haohui Mai |
+| [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree | Major | fairscheduler | Kenji Kikushima | Akira Ajisaka |
+| [HDFS-8135](https://issues.apache.org/jira/browse/HDFS-8135) | Remove the deprecated FSConstants class | Major | . | Haohui Mai | Li Lu |
+| [MAPREDUCE-6336](https://issues.apache.org/jira/browse/MAPREDUCE-6336) | Enable v2 FileOutputCommitter by default | Major | mrv2 | Gera Shegalov | Siqi Li |
+| [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | The RM should handle the graceful shutdown of the NM. | Major | nodemanager, resourcemanager | Ravi Teja Ch N V | Devaraj K |
+| [HDFS-8591](https://issues.apache.org/jira/browse/HDFS-8591) | Remove support for deprecated configuration key dfs.namenode.decommission.nodes.per.interval | Minor | namenode | Andrew Wang | Andrew Wang |
+| [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | Use slf4j instead of common-logging in hdfs-client | Major | build | Haohui Mai | Rakesh R |
+| [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | Fix typo in JobHistoryEventHandler | Minor | . | Brahma Reddy Battula | Ray Chiang |
+| [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3 | Major | fs/s3 | Thomas Demoor | Thomas Demoor |
+| [HDFS-8895](https://issues.apache.org/jira/browse/HDFS-8895) | Remove deprecated BlockStorageLocation APIs | Major | . | Andrew Wang | Andrew Wang |
+| [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | Compact XAttrs to optimize memory footprint. | Major | namenode | Yi Liu | Yi Liu |
+| [HDFS-8981](https://issues.apache.org/jira/browse/HDFS-8981) | Adding revision to data node jmx getVersion() method | Minor | . | Siqi Li | Siqi Li |
+| [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs | Major | . | Jian He | Jian He |
+| [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | Trash messages should be handled by Logger instead of being delivered on System.out | Major | trash | Ashutosh Chauhan | Mingliang Liu |
+| [HDFS-9085](https://issues.apache.org/jira/browse/HDFS-9085) | Show renewer information in DelegationTokenIdentifier#toString | Trivial | security | zhihai xu | zhihai xu |
+| [HADOOP-12495](https://issues.apache.org/jira/browse/HADOOP-12495) | Fix posix\_spawn error on OS X | Major | util | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-9070](https://issues.apache.org/jira/browse/HDFS-9070) | Allow fsck display pending replica location information for being-written blocks | Major | . | Rui Gao | Rui Gao |
+| [HDFS-9278](https://issues.apache.org/jira/browse/HDFS-9278) | Fix preferredBlockSize typo in OIV XML output | Trivial | tools | Nicole Pazmany | Nicole Pazmany |
+| [HADOOP-10787](https://issues.apache.org/jira/browse/HADOOP-10787) | Rename/remove non-HADOOP\_\*, etc from the shell scripts | Blocker | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12294](https://issues.apache.org/jira/browse/HADOOP-12294) | Throw an Exception when fs.permissions.umask-mode is misconfigured | Major | conf | Chang Li | Chang Li |
+| [HADOOP-10465](https://issues.apache.org/jira/browse/HADOOP-10465) | Fix use of generics within SortedMapWritable | Minor | . | Bertrand Dechoux | Bertrand Dechoux |
+| [HDFS-5165](https://issues.apache.org/jira/browse/HDFS-5165) | Remove the TotalFiles metrics | Minor | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | Retire libwebhdfs | Major | webhdfs | Allen Wittenauer | Haohui Mai |
+| [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | Replace dev-support with wrappers to Yetus | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6613](https://issues.apache.org/jira/browse/MAPREDUCE-6613) | Change mapreduce.jobhistory.jhist.format default from json to binary | Minor | . | Ray Chiang | Ray Chiang |
+| [HDFS-9395](https://issues.apache.org/jira/browse/HDFS-9395) | Make HDFS audit logging consistant | Major | . | Kihwal Lee | Kuhu Shukla |
+| [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | Fix undeclared/unused dependency to httpclient | Minor | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-12850](https://issues.apache.org/jira/browse/HADOOP-12850) | pull shell code out of hadoop-dist | Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | Remove all of the CHANGES.txt files | Major | build | Allen Wittenauer | Andrew Wang |
+| [HADOOP-12857](https://issues.apache.org/jira/browse/HADOOP-12857) | Rework hadoop-tools | Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12967](https://issues.apache.org/jira/browse/HADOOP-12967) | Remove FileUtil#copyMerge | Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HADOOP-12811](https://issues.apache.org/jira/browse/HADOOP-12811) | Change kms server port number which conflicts with HMaster port number | Critical | kms | Yufeng Jiang | Xiao Chen |
+| [YARN-4784](https://issues.apache.org/jira/browse/YARN-4784) | Fairscheduler: defaultQueueSchedulingPolicy should not accept FIFO | Major | fairscheduler | Yufei Gu | Yufei Gu |
+| [HDFS-9427](https://issues.apache.org/jira/browse/HDFS-9427) | HDFS should not default to ephemeral ports | Critical | datanode, hdfs-client, namenode | Arpit Agarwal | Xiaobing Zhou |
+| [HADOOP-12563](https://issues.apache.org/jira/browse/HADOOP-12563) | Updated utility to create/modify token files | Major | . | Allen Wittenauer | Matthew Paduano |
+| [HADOOP-13045](https://issues.apache.org/jira/browse/HADOOP-13045) | hadoop\_add\_classpath is not working in .hadooprc | Major | . | Akira Ajisaka | Allen Wittenauer |
+| [MAPREDUCE-6526](https://issues.apache.org/jira/browse/MAPREDUCE-6526) | Remove usage of metrics v1 from hadoop-mapreduce | Blocker | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-12504](https://issues.apache.org/jira/browse/HADOOP-12504) | Remove metrics v1 | Blocker | metrics | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-10337](https://issues.apache.org/jira/browse/HDFS-10337) | OfflineEditsViewer stats option should print 0 instead of null for the count of operations | Minor | . | Akira Ajisaka | Yiqun Lin |
+| [HADOOP-11858](https://issues.apache.org/jira/browse/HADOOP-11858) | [JDK8] Set minimum version of Hadoop 3 to JDK 8 | Blocker | build | Robert Kanter | Robert Kanter |
+| [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on | Major | fairscheduler | Karthik Kambatla | Karthik Kambatla |
+| [HADOOP-13175](https://issues.apache.org/jira/browse/HADOOP-13175) | Remove hadoop-ant from hadoop-tools | Major | . | Chris Douglas | Chris Douglas |
+| [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | fix/rewrite create-release | Blocker | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9613](https://issues.apache.org/jira/browse/HADOOP-9613) | [JDK8] Update jersey version to latest 1.x release | Major | build | Timothy St. Clair | Tsuyoshi Ozawa |
+| [HADOOP-13209](https://issues.apache.org/jira/browse/HADOOP-13209) | replace slaves with workers | Major | scripts | John Smith | John Smith |
+| [HDFS-6434](https://issues.apache.org/jira/browse/HDFS-6434) | Default permission for creating file should be 644 for WebHdfs/HttpFS | Minor | webhdfs | Juan Yu | Wellington Chevreuil |
+| [HADOOP-12864](https://issues.apache.org/jira/browse/HADOOP-12864) | Remove bin/rcc script | Blocker | tools | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12709](https://issues.apache.org/jira/browse/HADOOP-12709) | Cut s3:// from trunk | Major | fs/s3 | Steve Loughran | Mingliang Liu |
+| [HADOOP-12064](https://issues.apache.org/jira/browse/HADOOP-12064) | [JDK8] Update guice version to 4.0 | Blocker | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [HDFS-10548](https://issues.apache.org/jira/browse/HDFS-10548) | Remove the long deprecated BlockReaderRemote | Major | hdfs | Kai Zheng | Kai Zheng |
+| [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects | Major | build | Matt Foley | Matt Foley |
+| [HDFS-10689](https://issues.apache.org/jira/browse/HDFS-10689) | Hdfs dfs chmod should reset sticky bit permission when the bit is omitted in the octal mode | Minor | fs | Manoj Govindassamy | Manoj Govindassamy |
+| [HDFS-7933](https://issues.apache.org/jira/browse/HDFS-7933) | fsck should also report decommissioning replicas. | Major | namenode | Jitendra Nath Pandey | Xiaoyu Yao |
+| [HADOOP-12726](https://issues.apache.org/jira/browse/HADOOP-12726) | Unsupported FS operations should throw UnsupportedOperationException | Major | fs | Daniel Templeton | Daniel Templeton |
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler | Major | webapp | Jayesh | Varun Vasudev |
+| [HADOOP-11393](https://issues.apache.org/jira/browse/HADOOP-11393) | Revert HADOOP\_PREFIX, go back to HADOOP\_HOME | Major | scripts, tracing | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12930](https://issues.apache.org/jira/browse/HADOOP-12930) | [Umbrella] Dynamic subcommands for hadoop shell scripts | Critical | scripts | Allen Wittenauer | Allen Wittenauer |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-4887](https://issues.apache.org/jira/browse/MAPREDUCE-4887) | Rehashing partitioner for better distribution | Major | . | Radim Kolar | Radim Kolar |
+| [HADOOP-11013](https://issues.apache.org/jira/browse/HADOOP-11013) | CLASSPATH handling should be consolidated, debuggable | Major | . | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-6590](https://issues.apache.org/jira/browse/HADOOP-6590) | Add a username check for hadoop sub-commands | Minor | scripts | Scott Chen | John Smith |
+| [HADOOP-11353](https://issues.apache.org/jira/browse/HADOOP-11353) | Add support for .hadooprc | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9044](https://issues.apache.org/jira/browse/HADOOP-9044) | add FindClass main class to provide classpath checking of installations | Minor | util | Steve Loughran | Steve Loughran |
+| [HADOOP-11485](https://issues.apache.org/jira/browse/HADOOP-11485) | Pluggable shell integration | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | Shell command ls should include sort options | Minor | fs | Jonathan Allen | Jonathan Allen |
+| [HADOOP-11554](https://issues.apache.org/jira/browse/HADOOP-11554) | Expose HadoopKerberosName as a hadoop subcommand | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11565](https://issues.apache.org/jira/browse/HADOOP-11565) | Add --slaves shell option | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | Add posixGroups support for LDAP groups mapping service | Major | . | Kai Zheng | Dapeng Sun |
+| [HDFS-8009](https://issues.apache.org/jira/browse/HDFS-8009) | Signal congestion on the DataNode | Major | datanode | Haohui Mai | Haohui Mai |
+| [YARN-2901](https://issues.apache.org/jira/browse/YARN-2901) | Add errors and warning metrics page to RM, NM web UI | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [HADOOP-9642](https://issues.apache.org/jira/browse/HADOOP-9642) | Configuration to resolve environment variables via ${env.VARIABLE} references | Minor | conf, scripts | Steve Loughran | Kengo Seki |
+| [HDFS-7891](https://issues.apache.org/jira/browse/HDFS-7891) | A block placement policy with best rack failure tolerance | Minor | namenode | Walter Su | Walter Su |
+| [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | Make setting up the build environment easier | Major | build | Niels Basjes | Niels Basjes |
+| [MAPREDUCE-6284](https://issues.apache.org/jira/browse/MAPREDUCE-6284) | Add Task Attempt State API to MapReduce Application Master REST API | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-10971](https://issues.apache.org/jira/browse/HADOOP-10971) | Add -C flag to make `hadoop fs -ls` print filenames only | Major | fs | Ryan Williams | Kengo Seki |
+| [MAPREDUCE-6364](https://issues.apache.org/jira/browse/MAPREDUCE-6364) | Add a "Kill" link to Task Attempts page | Minor | applicationmaster | Ryu Kobayashi | Ryu Kobayashi |
+| [MAPREDUCE-6304](https://issues.apache.org/jira/browse/MAPREDUCE-6304) | Specifying node labels when submitting MR jobs | Major | . | Jian Fang | Naganarasimha G R |
+| [HADOOP-7947](https://issues.apache.org/jira/browse/HADOOP-7947) | Validate XMLs if a relevant tool is available, when using scripts | Major | scripts | Harsh J | Kengo Seki |
+| [HDFS-8487](https://issues.apache.org/jira/browse/HDFS-8487) | Generalize BlockInfo in preparation of merging HDFS-7285 into trunk and branch-2 | Major | namenode | Zhe Zhang | Zhe Zhang |
+| [HDFS-8608](https://issues.apache.org/jira/browse/HDFS-8608) | Merge HDFS-7912 to trunk and branch-2 (track BlockInfo instead of Block in UnderReplicatedBlocks and PendingReplicationBlocks) | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-6440](https://issues.apache.org/jira/browse/HDFS-6440) | Support more than 2 NameNodes | Major | auto-failover, ha, namenode | Jesse Yates | Jesse Yates |
+| [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | Add SFTP FileSystem | Minor | fs | ��igo Goiri | ramtin |
+| [HDFS-8622](https://issues.apache.org/jira/browse/HDFS-8622) | Implement GETCONTENTSUMMARY operation for WebImageViewer | Major | . | Jagadesh Kiran N | Jagadesh Kiran N |
+| [HDFS-8155](https://issues.apache.org/jira/browse/HDFS-8155) | Support OAuth2 in WebHDFS | Major | webhdfs | Jakob Homan | Jakob Homan |
+| [MAPREDUCE-6415](https://issues.apache.org/jira/browse/MAPREDUCE-6415) | Create a tool to combine aggregated logs into HAR files | Major | . | Robert Kanter | Robert Kanter |
+| [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | Create StatsD metrics2 sink | Minor | metrics | Dave Marion | Dave Marion |
+| [HDFS-7285](https://issues.apache.org/jira/browse/HDFS-7285) | Erasure Coding Support inside HDFS | Major | . | Weihua Jiang | Zhe Zhang |
+| [HADOOP-10949](https://issues.apache.org/jira/browse/HADOOP-10949) | metrics2 sink plugin for Apache Kafka | Major | metrics | Babak Behzad | Babak Behzad |
+| [YARN-261](https://issues.apache.org/jira/browse/YARN-261) | Ability to fail AM attempts | Major | api | Jason Lowe | Rohith Sharma K S |
+| [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | Logging HDFS operation's caller context into audit logs | Major | . | Mingliang Liu | Mingliang Liu |
+| [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | allow/disallow snapshots via webhdfs | Major | webhdfs | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-4349](https://issues.apache.org/jira/browse/YARN-4349) | Support CallerContext in YARN | Major | . | Wangda Tan | Wangda Tan |
+| [HADOOP-12366](https://issues.apache.org/jira/browse/HADOOP-12366) | expose calculated paths | Major | . | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12321](https://issues.apache.org/jira/browse/HADOOP-12321) | Make JvmPauseMonitor an AbstractService | Major | . | Steve Loughran | Sunil G |
+| [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | We should have a config to indicate the Timeline Service version | Major | timelineserver | Zhijie Shen | Xuan Gong |
+| [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | Add a option to skip newline on empty files with getMerge -nl | Minor | . | Jan Filipiak | Kanaka Kumar Avvaru |
+| [YARN-3458](https://issues.apache.org/jira/browse/YARN-3458) | CPU resource monitoring in Windows | Minor | nodemanager | Inigo Goiri | Inigo Goiri |
+| [HADOOP-12691](https://issues.apache.org/jira/browse/HADOOP-12691) | Add CSRF Filter for REST APIs to Hadoop Common | Major | security | Larry McCay | Larry McCay |
+| [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | Adding Append API support for WASB | Major | azure | Dushyanth | Dushyanth |
+| [HDFS-9525](https://issues.apache.org/jira/browse/HDFS-9525) | hadoop utilities need to support provided delegation tokens | Blocker | security | Allen Wittenauer | HeeSoo Kim |
+| [HADOOP-12702](https://issues.apache.org/jira/browse/HADOOP-12702) | Add an HDFS metrics sink | Major | metrics | Daniel Templeton | Daniel Templeton |
+| [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | Add Entry point for Kerberos health check | Minor | security | Steve Loughran | Steve Loughran |
+| [HDFS-9244](https://issues.apache.org/jira/browse/HDFS-9244) | Support nested encryption zones | Major | encryption | Xiaoyu Yao | Zhe Zhang |
+| [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | Read s3a creds from a Credential Provider | Major | fs/s3 | Allen Wittenauer | Larry McCay |
+| [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | Integrate CSRF prevention filter in WebHDFS. | Major | datanode, namenode, webhdfs | Chris Nauroth | Chris Nauroth |
+| [HDFS-9835](https://issues.apache.org/jira/browse/HDFS-9835) | OIV: add ReverseXML processor which reconstructs an fsimage from an XML file | Major | tools | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness | Major | datanode, namenode | Chris Nauroth | Chris Nauroth |
+| [HADOOP-12909](https://issues.apache.org/jira/browse/HADOOP-12909) | Change ipc.Client to support asynchronous calls | Major | ipc | Tsz Wo Nicholas Sze | Xiaobing Zhou |
+| [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | Datanode command for evicting writers | Major | datanode | Kihwal Lee | Kihwal Lee |
+| [HADOOP-13008](https://issues.apache.org/jira/browse/HADOOP-13008) | Add XFS Filter for UIs to Hadoop Common | Major | security | Larry McCay | Larry McCay |
+| [HADOOP-13065](https://issues.apache.org/jira/browse/HADOOP-13065) | Add a new interface for retrieving FS and FC Statistics | Major | fs | Ram Venkatesh | Mingliang Liu |
+| [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | S3A: Add ability to plug in any AWSCredentialsProvider | Major | fs/s3 | Steven K. Wong | Steven K. Wong |
+| [HADOOP-12847](https://issues.apache.org/jira/browse/HADOOP-12847) | hadoop daemonlog should support https and SPNEGO for Kerberized cluster | Major | security | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-13226](https://issues.apache.org/jira/browse/HADOOP-13226) | Support async call retry and failover | Major | io, ipc | Xiaobing Zhou | Tsz Wo Nicholas Sze |
+| [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | S3A to support Amazon STS temporary credentials | Minor | fs/s3 | Sean Mackrory | Sean Mackrory |
+| [HADOOP-12666](https://issues.apache.org/jira/browse/HADOOP-12666) | Support Microsoft Azure Data Lake - as a file system in Hadoop | Major | fs, fs/azure, tools | Vishwajeet Dusane | Vishwajeet Dusane |
+| [HDFS-1312](https://issues.apache.org/jira/browse/HDFS-1312) | Re-balance disks within a Datanode | Major | datanode | Travis Crawford | Anu Engineer |
+| [YARN-2928](https://issues.apache.org/jira/browse/YARN-2928) | YARN Timeline Service v.2: alpha 1 | Critical | timelineserver | Sangjin Lee | Sangjin Lee |
+| [MAPREDUCE-6331](https://issues.apache.org/jira/browse/MAPREDUCE-6331) | [Umbrella] Make MapReduce work with Timeline Service Nextgen (YARN-2928) | Major | . | Vinod Kumar Vavilapalli | Sangjin Lee |
+| [HADOOP-12747](https://issues.apache.org/jira/browse/HADOOP-12747) | support wildcard in libjars argument | Major | util | Sangjin Lee | Sangjin Lee |
+| [HDFS-9804](https://issues.apache.org/jira/browse/HDFS-9804) | Allow long-running Balancer to login with keytab | Major | balancer & mover, security | Xiao Chen | Xiao Chen |
+| [MAPREDUCE-6690](https://issues.apache.org/jira/browse/MAPREDUCE-6690) | Limit the number of resources a single map reduce job can submit for localization | Major | . | Chris Trezzo | Chris Trezzo |
+| [MAPREDUCE-6578](https://issues.apache.org/jira/browse/MAPREDUCE-6578) | Add support for HDFS heterogeneous storage testing to TestDFSIO | Major | . | Wei Zhou | Wei Zhou |
+| [HADOOP-13396](https://issues.apache.org/jira/browse/HADOOP-13396) | Allow pluggable audit loggers in KMS | Major | kms | Xiao Chen | Xiao Chen |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-2944](https://issues.apache.org/jira/browse/MAPREDUCE-2944) | Improve checking of input for Api displayTasks() | Trivial | client | XieXianshan | XieXianshan |
+| [HADOOP-8017](https://issues.apache.org/jira/browse/HADOOP-8017) | Configure hadoop-main pom to get rid of M2E plugin execution not covered | Major | build | Eric Charles | Eric Charles |
+| [HADOOP-8078](https://issues.apache.org/jira/browse/HADOOP-8078) | Add capability to turn on security in unit tests. | Major | test | Jitendra Nath Pandey | Jaimin D Jetly |
+| [MAPREDUCE-3956](https://issues.apache.org/jira/browse/MAPREDUCE-3956) | Remove the use of the deprecated Syncable.sync() method | Minor | examples | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HADOOP-8147](https://issues.apache.org/jira/browse/HADOOP-8147) | test-patch should run tests with -fn to avoid masking test failures | Major | test | Robert Joseph Evans | Robert Joseph Evans |
+| [HADOOP-8290](https://issues.apache.org/jira/browse/HADOOP-8290) | Remove two remaining references to "hadoop.native.lib" oldprop | Trivial | test | Harsh J | Harsh J |
+| [HDFS-2743](https://issues.apache.org/jira/browse/HDFS-2743) | Streamline usage of bookkeeper journal manager | Major | ha | Ivan Kelly | Ivan Kelly |
+| [HADOOP-8312](https://issues.apache.org/jira/browse/HADOOP-8312) | testpatch.sh should provide a simpler way to see which warnings changed | Major | scripts | Robert Joseph Evans | Robert Joseph Evans |
+| [HDFS-3293](https://issues.apache.org/jira/browse/HDFS-3293) | Implement equals for journainfo class. | Minor | namenode | Hari Mankude | Hari Mankude |
+| [HADOOP-8375](https://issues.apache.org/jira/browse/HADOOP-8375) | test-patch should stop immediately once it has found compilation errors | Major | build | Tsz Wo Nicholas Sze | Robert Joseph Evans |
+| [HADOOP-8297](https://issues.apache.org/jira/browse/HADOOP-8297) | Writable javadocs don't carry default constructor | Trivial | documentation | Harsh J | Harsh J |
+| [HADOOP-7659](https://issues.apache.org/jira/browse/HADOOP-7659) | fs -getmerge isn't guaranteed to work well over non-HDFS filesystems | Minor | fs | Harsh J | Harsh J |
+| [MAPREDUCE-3302](https://issues.apache.org/jira/browse/MAPREDUCE-3302) | Remove the last dependency call from org.apache.hadoop.record package in MR. | Minor | client | Harsh J | Harsh J |
+| [MAPREDUCE-2384](https://issues.apache.org/jira/browse/MAPREDUCE-2384) | The job submitter should make sure to validate jobs before creation of necessary files | Major | job submission, test | Denny Ye | Harsh J |
+| [HDFS-3476](https://issues.apache.org/jira/browse/HDFS-3476) | Correct the default used in TestDFSClientRetries.busyTest() after HDFS-3462 | Minor | test | Harsh J | Harsh J |
+| [HDFS-3478](https://issues.apache.org/jira/browse/HDFS-3478) | Test quotas with Long.Max\_Value | Minor | namenode | Sujay Rau | Sujay Rau |
+| [MAPREDUCE-4371](https://issues.apache.org/jira/browse/MAPREDUCE-4371) | Check for cyclic dependencies in Jobcontrol job DAG | Major | mrv1 | madhukara phatak | madhukara phatak |
+| [HADOOP-8523](https://issues.apache.org/jira/browse/HADOOP-8523) | test-patch.sh doesn't validate patches before building | Minor | build | Jack Dintruff | Jack Dintruff |
+| [HADOOP-8521](https://issues.apache.org/jira/browse/HADOOP-8521) | Port StreamInputFormat to new Map Reduce API | Major | streaming | madhukara phatak | madhukara phatak |
+| [HADOOP-8593](https://issues.apache.org/jira/browse/HADOOP-8593) | add the missed @Override to methods in Metric/Metric2 package | Minor | metrics | Brandon Li | Brandon Li |
+| [HDFS-3768](https://issues.apache.org/jira/browse/HDFS-3768) | Exception in TestJettyHelper is incorrect | Minor | test | Jakob Homan | Eli Reisman |
+| [HDFS-3834](https://issues.apache.org/jira/browse/HDFS-3834) | Unused Command.java fields NAME, DESCRIPTION and USAGE | Minor | shell | Jing Zhao | Jing Zhao |
+| [HDFS-2580](https://issues.apache.org/jira/browse/HDFS-2580) | NameNode#main(...) can make use of GenericOptionsParser. | Minor | namenode | Harsh J | Harsh J |
+| [HADOOP-8588](https://issues.apache.org/jira/browse/HADOOP-8588) | SerializationFactory shouldn't throw a NullPointerException if the serializations list is empty | Minor | io | Harsh J | Sho Shimauchi |
+| [HADOOP-7930](https://issues.apache.org/jira/browse/HADOOP-7930) | Kerberos relogin interval in UserGroupInformation should be configurable | Major | security | Alejandro Abdelnur | Robert Kanter |
+| [HADOOP-8838](https://issues.apache.org/jira/browse/HADOOP-8838) | Colorize the test-patch output sent to JIRA | Trivial | build | Harsh J | Harsh J |
+| [MAPREDUCE-4686](https://issues.apache.org/jira/browse/MAPREDUCE-4686) | hadoop-mapreduce-client-core fails compilation in Eclipse due to missing Avro-generated classes | Major | build | Chris Nauroth | Chris Nauroth |
+| [HADOOP-8864](https://issues.apache.org/jira/browse/HADOOP-8864) | Addendum to HADOOP-8840: Add a coloring case for +0 results too. | Trivial | precommit | Harsh J | Harsh J |
+| [HDFS-3995](https://issues.apache.org/jira/browse/HDFS-3995) | Use DFSTestUtil.createFile() for file creation and writing in test cases | Minor | test | Jing Zhao | Jing Zhao |
+| [HADOOP-8920](https://issues.apache.org/jira/browse/HADOOP-8920) | Add more javadoc to metrics2 related classes | Minor | metrics | Suresh Srinivas | Suresh Srinivas |
+| [HADOOP-8776](https://issues.apache.org/jira/browse/HADOOP-8776) | Provide an option in test-patch that can enable / disable compiling native code | Minor | build | Hemanth Yamijala | Chris Nauroth |
+| [MAPREDUCE-4735](https://issues.apache.org/jira/browse/MAPREDUCE-4735) | Make arguments in TestDFSIO case insensitive | Trivial | test | Robert Kanter | Brandon Li |
+| [HADOOP-6616](https://issues.apache.org/jira/browse/HADOOP-6616) | Improve documentation for rack awareness | Major | documentation | Jeff Hammerbacher | Adam Faris |
+| [HADOOP-9075](https://issues.apache.org/jira/browse/HADOOP-9075) | FileContext#FSLinkResolver should be made static | Minor | . | Suresh Srinivas | Arpit Agarwal |
+| [HDFS-3358](https://issues.apache.org/jira/browse/HDFS-3358) | Specify explicitly that the NN UI status total is talking of persistent objects on heap. | Trivial | namenode | Harsh J | Harsh J |
+| [HADOOP-9277](https://issues.apache.org/jira/browse/HADOOP-9277) | Improve javadoc for FileContext | Minor | . | Andrew Wang | Andrew Wang |
+| [HADOOP-9394](https://issues.apache.org/jira/browse/HADOOP-9394) | Port findHangingTest.sh from HBase to Hadoop | Minor | . | Andrew Wang | Andrew Wang |
+| [HDFS-4665](https://issues.apache.org/jira/browse/HDFS-4665) | Move TestNetworkTopologyWithNodeGroup to Common | Minor | test | Luke Lu | Junping Du |
+| [MAPREDUCE-5232](https://issues.apache.org/jira/browse/MAPREDUCE-5232) | log classpath and other key properties on child JVM start | Major | mrv1, mrv2 | Sangjin Lee | Sangjin Lee |
+| [HDFS-5041](https://issues.apache.org/jira/browse/HDFS-5041) | Add the time of last heartbeat to dead server Web UI | Minor | . | Ted Yu | Ted Yu |
+| [HADOOP-10044](https://issues.apache.org/jira/browse/HADOOP-10044) | Improve the javadoc of rpc code | Minor | . | Sanjay Radia | Sanjay Radia |
+| [MAPREDUCE-5197](https://issues.apache.org/jira/browse/MAPREDUCE-5197) | Checkpoint Service: a library component to facilitate checkpoint of task state | Major | mrv2 | Carlo Curino | Carlo Curino |
+| [MAPREDUCE-5196](https://issues.apache.org/jira/browse/MAPREDUCE-5196) | CheckpointAMPreemptionPolicy implements preemption in MR AM via checkpointing | Major | mr-am, mrv2 | Carlo Curino | Carlo Curino |
+| [HDFS-6228](https://issues.apache.org/jira/browse/HDFS-6228) | comments typo fix for FsDatasetImpl.java | Trivial | . | yunjiong zhao | yunjiong zhao |
+| [HADOOP-8719](https://issues.apache.org/jira/browse/HADOOP-8719) | Workaround for kerberos-related log errors upon running any hadoop command on OSX | Trivial | scripts | Jianbin Wei | Jianbin Wei |
+| [HADOOP-10834](https://issues.apache.org/jira/browse/HADOOP-10834) | Typo in CredentialShell usage | Minor | security | Benoy Antony | Benoy Antony |
+| [MAPREDUCE-6013](https://issues.apache.org/jira/browse/MAPREDUCE-6013) | mapred version is missing | Major | scripts | Allen Wittenauer | Akira Ajisaka |
+| [MAPREDUCE-2841](https://issues.apache.org/jira/browse/MAPREDUCE-2841) | Task level native optimization | Major | task | Binglin Chang | Sean Zhong |
+| [YARN-2438](https://issues.apache.org/jira/browse/YARN-2438) | yarn-env.sh cleanup | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11092](https://issues.apache.org/jira/browse/HADOOP-11092) | hadoop shell commands should print usage if not given a class | Major | scripts | Bruno Mah� | Allen Wittenauer |
+| [HDFS-7278](https://issues.apache.org/jira/browse/HDFS-7278) | Add a command that allows sysadmins to manually trigger full block reports from a DN | Major | datanode | Colin P. McCabe | Colin P. McCabe |
+| [HADOOP-11025](https://issues.apache.org/jira/browse/HADOOP-11025) | hadoop-daemons.sh should just call hdfs directly | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [HADOOP-11150](https://issues.apache.org/jira/browse/HADOOP-11150) | hadoop command should show the reason on failure by invalid COMMAND or CLASSNAME | Minor | scripts | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-11208](https://issues.apache.org/jira/browse/HADOOP-11208) | Replace "daemon" with better name in script subcommands | Major | scripts | Yongjun Zhang | Allen Wittenauer |
+| [HADOOP-10926](https://issues.apache.org/jira/browse/HADOOP-10926) | Improve smart-apply-patch.sh to apply binary diffs | Major | . | Andrew Wang | Colin P. McCabe |
+| [HADOOP-11081](https://issues.apache.org/jira/browse/HADOOP-11081) | Document hadoop properties expected to be set by the shell code in \*-env.sh | Major | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [YARN-2472](https://issues.apache.org/jira/browse/YARN-2472) | yarn-daemons.sh should just call yarn directly | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [YARN-2301](https://issues.apache.org/jira/browse/YARN-2301) | Improve yarn container command | Major | . | Jian He | Naganarasimha G R |
+| [HADOOP-11352](https://issues.apache.org/jira/browse/HADOOP-11352) | Clean up test-patch.sh to disable "+1 contrib tests" | Major | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-2437](https://issues.apache.org/jira/browse/YARN-2437) | start-yarn.sh/stop-yarn should give info | Major | scripts | Allen Wittenauer | Varun Saxena |
+| [HDFS-7531](https://issues.apache.org/jira/browse/HDFS-7531) | Improve the concurrent access on FsVolumeList | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-11058](https://issues.apache.org/jira/browse/HADOOP-11058) | Missing HADOOP\_CONF\_DIR generates strange results | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [HDFS-7579](https://issues.apache.org/jira/browse/HDFS-7579) | Improve log reporting during block report rpc failure | Minor | datanode | Charles Lamb | Charles Lamb |
+| [HDFS-7446](https://issues.apache.org/jira/browse/HDFS-7446) | HDFS inotify should have the ability to determine what txid it has read up to | Major | hdfs-client | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-7182](https://issues.apache.org/jira/browse/HDFS-7182) | JMX metrics aren't accessible when NN is busy | Major | . | Ming Ma | Ming Ma |
+| [HADOOP-11466](https://issues.apache.org/jira/browse/HADOOP-11466) | FastByteComparisons: do not use UNSAFE\_COMPARER on the SPARC architecture because it is slower there | Minor | io, performance, util | Suman Somasundar | Suman Somasundar |
+| [HADOOP-11460](https://issues.apache.org/jira/browse/HADOOP-11460) | Deprecate shell vars | Major | scripts | Allen Wittenauer | John Smith |
+| [HADOOP-11346](https://issues.apache.org/jira/browse/HADOOP-11346) | Rewrite sls/rumen to use new shell framework | Major | scripts, tools | Allen Wittenauer | John Smith |
+| [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | dfs -count -q should label output column | Trivial | . | Nigel Daley | Jonathan Allen |
+| [HADOOP-6964](https://issues.apache.org/jira/browse/HADOOP-6964) | Allow compact property description in xml | Major | conf | Owen O'Malley | Kengo Seki |
+| [HADOOP-11506](https://issues.apache.org/jira/browse/HADOOP-11506) | Configuration variable expansion regex expensive for long values | Major | conf | Dmitriy V. Ryaboy | Gera Shegalov |
+| [HDFS-7546](https://issues.apache.org/jira/browse/HDFS-7546) | Document, and set an accepting default for dfs.namenode.kerberos.principal.pattern | Minor | security | Harsh J | Harsh J |
+| [HDFS-7694](https://issues.apache.org/jira/browse/HDFS-7694) | FSDataInputStream should support "unbuffer" | Major | . | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-7322](https://issues.apache.org/jira/browse/HDFS-7322) | deprecate sbin/hadoop-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [YARN-2796](https://issues.apache.org/jira/browse/YARN-2796) | deprecate sbin/yarn-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6250](https://issues.apache.org/jira/browse/MAPREDUCE-6250) | deprecate sbin/mr-jobhistory-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6260](https://issues.apache.org/jira/browse/MAPREDUCE-6260) | Convert site documentation to markdown | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [HADOOP-11575](https://issues.apache.org/jira/browse/HADOOP-11575) | Daemon log documentation is misleading | Major | documentation | Jagadesh Kiran N | Naganarasimha G R |
+| [HADOOP-11596](https://issues.apache.org/jira/browse/HADOOP-11596) | Allow smart-apply-patch.sh to add new files in binary git patches | Major | . | Ravi Prakash | Ravi Prakash |
+| [HADOOP-11593](https://issues.apache.org/jira/browse/HADOOP-11593) | Convert site documentation from apt to markdown (stragglers) | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [YARN-3230](https://issues.apache.org/jira/browse/YARN-3230) | Clarify application states on the web UI | Major | . | Jian He | Jian He |
+| [HDFS-7460](https://issues.apache.org/jira/browse/HDFS-7460) | Rewrite httpfs to use new shell framework | Major | scripts | Allen Wittenauer | John Smith |
+| [YARN-3168](https://issues.apache.org/jira/browse/YARN-3168) | Convert site documentation from apt to markdown | Major | documentation | Allen Wittenauer | Gururaj Shetty |
+| [YARN-3199](https://issues.apache.org/jira/browse/YARN-3199) | Fair Scheduler documentation improvements | Minor | fairscheduler | Rohit Agarwal | Rohit Agarwal |
+| [MAPREDUCE-6267](https://issues.apache.org/jira/browse/MAPREDUCE-6267) | Refactor JobSubmitter#copyAndConfigureFiles into it's own class | Minor | . | Chris Trezzo | Chris Trezzo |
+| [YARN-3249](https://issues.apache.org/jira/browse/YARN-3249) | Add a "kill application" button to Resource Manager's Web UI | Minor | resourcemanager | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-11673](https://issues.apache.org/jira/browse/HADOOP-11673) | Use org.junit.Assume to skip tests instead of return | Minor | test | Akira Ajisaka | Brahma Reddy Battula |
+| [HADOOP-11692](https://issues.apache.org/jira/browse/HADOOP-11692) | Improve authentication failure WARN message to avoid user confusion | Major | ipc | Yongjun Zhang | Yongjun Zhang |
+| [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY | Major | ipc | Gopal V | Gopal V |
+| [YARN-2280](https://issues.apache.org/jira/browse/YARN-2280) | Resource manager web service fields are not accessible | Trivial | resourcemanager | Krisztian Horvath | Krisztian Horvath |
+| [HADOOP-11711](https://issues.apache.org/jira/browse/HADOOP-11711) | Provide a default value for AES/CTR/NoPadding CryptoCodec classes | Minor | . | Andrew Wang | Andrew Wang |
+| [MAPREDUCE-4414](https://issues.apache.org/jira/browse/MAPREDUCE-4414) | Add main methods to JobConf and YarnConfiguration, for debug purposes | Major | client | Harsh J | Plamen Jeliazkov |
+| [MAPREDUCE-6105](https://issues.apache.org/jira/browse/MAPREDUCE-6105) | Inconsistent configuration in property mapreduce.reduce.shuffle.merge.percent | Trivial | . | Dongwook Kwon | Ray Chiang |
+| [HDFS-2360](https://issues.apache.org/jira/browse/HDFS-2360) | Ugly stacktrace when quota exceeds | Minor | hdfs-client | Rajit Saha | Harsh J |
+| [MAPREDUCE-6100](https://issues.apache.org/jira/browse/MAPREDUCE-6100) | replace "mapreduce.job.credentials.binary" with MRJobConfig.MAPREDUCE\_JOB\_CREDENTIALS\_BINARY for better readability. | Trivial | mrv2 | zhihai xu | zhihai xu |
+| [MAPREDUCE-5755](https://issues.apache.org/jira/browse/MAPREDUCE-5755) | MapTask.MapOutputBuffer#compare/swap should have @Override annotation | Trivial | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [MAPREDUCE-4653](https://issues.apache.org/jira/browse/MAPREDUCE-4653) | TestRandomAlgorithm has an unused "import" statement | Trivial | contrib/gridmix | Amir Sanjar | Amir Sanjar |
+| [HADOOP-11659](https://issues.apache.org/jira/browse/HADOOP-11659) | o.a.h.fs.FileSystem.Cache#remove should use a single hash map lookup | Minor | fs | Gera Shegalov | Brahma Reddy Battula |
+| [HADOOP-11709](https://issues.apache.org/jira/browse/HADOOP-11709) | Time.NANOSECONDS\_PER\_MILLISECOND - use class-level final constant instead of method variable | Trivial | . | Ajith S | Ajith S |
+| [HDFS-7835](https://issues.apache.org/jira/browse/HDFS-7835) | make initial sleeptime in locateFollowingBlock configurable for DFSClient. | Major | hdfs-client | zhihai xu | zhihai xu |
+| [HDFS-7829](https://issues.apache.org/jira/browse/HDFS-7829) | Code clean up for LocatedBlock | Minor | . | Jing Zhao | Takanobu Asanuma |
+| [MAPREDUCE-6282](https://issues.apache.org/jira/browse/MAPREDUCE-6282) | Reuse historyFileAbsolute.getFileSystem in CompletedJob#loadFullHistoryData for code optimization. | Trivial | jobhistoryserver | zhihai xu | zhihai xu |
+| [HADOOP-11447](https://issues.apache.org/jira/browse/HADOOP-11447) | Add a more meaningful toString method to SampleStat and MutableStat | Minor | metrics | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3350](https://issues.apache.org/jira/browse/YARN-3350) | YARN RackResolver spams logs with messages at info level | Major | . | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [MAPREDUCE-6239](https://issues.apache.org/jira/browse/MAPREDUCE-6239) | Consolidate TestJobConf classes in hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core | Minor | client | Varun Saxena | Varun Saxena |
+| [MAPREDUCE-5190](https://issues.apache.org/jira/browse/MAPREDUCE-5190) | Unnecessary condition test in RandomSampler | Minor | mrv2 | Jingguo Yao | Jingguo Yao |
+| [MAPREDUCE-6287](https://issues.apache.org/jira/browse/MAPREDUCE-6287) | Deprecated methods in org.apache.hadoop.examples.Sort | Minor | examples | Chao Zhang | Chao Zhang |
+| [HADOOP-11737](https://issues.apache.org/jira/browse/HADOOP-11737) | mockito's version in hadoop-nfs\u2019 pom.xml shouldn't be specified | Minor | nfs | Kengo Seki | Kengo Seki |
+| [YARN-2868](https://issues.apache.org/jira/browse/YARN-2868) | FairScheduler: Metric for latency to allocate first container for an application | Major | . | Ray Chiang | Ray Chiang |
+| [HDFS-7875](https://issues.apache.org/jira/browse/HDFS-7875) | Improve log message when wrong value configured for dfs.datanode.failed.volumes.tolerated | Trivial | datanode | nijel | nijel |
+| [HDFS-7793](https://issues.apache.org/jira/browse/HDFS-7793) | Refactor DFSOutputStream separating DataStreamer out | Major | hdfs-client | Kai Zheng | Li Bo |
+| [HADOOP-11741](https://issues.apache.org/jira/browse/HADOOP-11741) | Add LOG.isDebugEnabled() guard for some LOG.debug() | Major | . | Walter Su | Walter Su |
+| [MAPREDUCE-579](https://issues.apache.org/jira/browse/MAPREDUCE-579) | Streaming "slowmatch" documentation | Trivial | contrib/streaming | Bo Adler | Harsh J |
+| [HDFS-7928](https://issues.apache.org/jira/browse/HDFS-7928) | Scanning blocks from disk during rolling upgrade startup takes a lot of time if disks are busy | Major | datanode | Rushabh S Shah | Rushabh S Shah |
+| [HADOOP-11524](https://issues.apache.org/jira/browse/HADOOP-11524) | hadoop\_do\_classpath\_subcommand throws a shellcheck warning | Minor | scripts | Allen Wittenauer | Chris Nauroth |
+| [HADOOP-11719](https://issues.apache.org/jira/browse/HADOOP-11719) | [Fsshell] Remove bin/hadoop reference from GenericOptionsParser default help text | Minor | scripts | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8004](https://issues.apache.org/jira/browse/HDFS-8004) | Use KeyProviderCryptoExtension#warmUpEncryptedKeys when creating an encryption zone | Trivial | encryption | Andrew Wang | Andrew Wang |
+| [HDFS-7890](https://issues.apache.org/jira/browse/HDFS-7890) | Improve information on Top users for metrics in RollingWindowsManager and lower log level | Major | . | J.Andreina | J.Andreina |
+| [HDFS-4396](https://issues.apache.org/jira/browse/HDFS-4396) | Add START\_MSG/SHUTDOWN\_MSG for ZKFC | Major | auto-failover, ha, tools | Liang Xie | Liang Xie |
+| [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | Add support for hardware crc of HDFS checksums on ARM aarch64 architecture | Minor | native | Edward Nevill | Edward Nevill |
+| [HDFS-7645](https://issues.apache.org/jira/browse/HDFS-7645) | Rolling upgrade is restoring blocks from trash multiple times | Major | datanode | Nathan Roberts | Keisuke Ogiwara |
+| [HDFS-3918](https://issues.apache.org/jira/browse/HDFS-3918) | EditLogTailer shouldn't log WARN when other node is in standby mode | Major | ha | Todd Lipcon | Todd Lipcon |
+| [HDFS-7944](https://issues.apache.org/jira/browse/HDFS-7944) | Minor cleanup of BlockPoolManager#getAllNamenodeThreads | Minor | . | Arpit Agarwal | Arpit Agarwal |
+| [YARN-3258](https://issues.apache.org/jira/browse/YARN-3258) | FairScheduler: Need to add more logging to investigate allocations | Minor | fairscheduler | Anubhav Dhoot | Anubhav Dhoot |
+| [HDFS-7671](https://issues.apache.org/jira/browse/HDFS-7671) | hdfs user guide should point to the common rack awareness doc | Major | documentation | Allen Wittenauer | Kai Sasaki |
+| [YARN-3412](https://issues.apache.org/jira/browse/YARN-3412) | RM tests should use MockRM where possible | Major | resourcemanager, test | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3428](https://issues.apache.org/jira/browse/YARN-3428) | Debug log resources to be localized for a container | Trivial | nodemanager | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3424](https://issues.apache.org/jira/browse/YARN-3424) | Change logs for ContainerMonitorImpl's resourse monitoring from info to debug | Major | nodemanager | Anubhav Dhoot | Anubhav Dhoot |
+| [YARN-3248](https://issues.apache.org/jira/browse/YARN-3248) | Display count of nodes blacklisted by apps in the web UI | Major | capacityscheduler, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [HDFS-7978](https://issues.apache.org/jira/browse/HDFS-7978) | Add LOG.isDebugEnabled() guard for some LOG.debug(..) | Major | . | Walter Su | Walter Su |
+| [HDFS-8008](https://issues.apache.org/jira/browse/HDFS-8008) | Support client-side back off when the datanodes are congested | Major | hdfs-client | Haohui Mai | Haohui Mai |
+| [HDFS-7888](https://issues.apache.org/jira/browse/HDFS-7888) | Change DataStreamer/DFSOutputStream/DFSPacket for convenience of subclassing | Minor | hdfs-client | Li Bo | Li Bo |
+| [HDFS-8035](https://issues.apache.org/jira/browse/HDFS-8035) | Move checkBlocksProperlyReplicated() in FSNamesystem to BlockManager | Minor | namenode | Haohui Mai | Haohui Mai |
+| [HADOOP-9805](https://issues.apache.org/jira/browse/HADOOP-9805) | Refactor RawLocalFileSystem#rename for improved testability. | Minor | fs, test | Chris Nauroth | Jean-Pierre Matsumoto |
+| [HADOOP-11785](https://issues.apache.org/jira/browse/HADOOP-11785) | Reduce number of listStatus operation in distcp buildListing() | Minor | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [HADOOP-11717](https://issues.apache.org/jira/browse/HADOOP-11717) | Add Redirecting WebSSO behavior with JWT Token in Hadoop Auth | Major | security | Larry McCay | Larry McCay |
+| [YARN-3294](https://issues.apache.org/jira/browse/YARN-3294) | Allow dumping of Capacity Scheduler debug logs via web UI for a fixed time period | Major | capacityscheduler | Varun Vasudev | Varun Vasudev |
+| [HDFS-8073](https://issues.apache.org/jira/browse/HDFS-8073) | Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided. | Trivial | namenode | Walter Su | Walter Su |
+| [HDFS-8076](https://issues.apache.org/jira/browse/HDFS-8076) | Code cleanup for DFSInputStream: use offset instead of LocatedBlock when possible | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-7979](https://issues.apache.org/jira/browse/HDFS-7979) | Initialize block report IDs with a random number | Minor | datanode | Andrew Wang | Andrew Wang |
+| [HDFS-8101](https://issues.apache.org/jira/browse/HDFS-8101) | DFSClient use of non-constant DFSConfigKeys pulls in WebHDFS classes at runtime | Minor | hdfs-client | Sean Busbey | Sean Busbey |
+| [YARN-3293](https://issues.apache.org/jira/browse/YARN-3293) | Track and display capacity scheduler health metrics in web UI | Major | capacityscheduler | Varun Vasudev | Varun Vasudev |
+| [MAPREDUCE-6291](https://issues.apache.org/jira/browse/MAPREDUCE-6291) | Correct mapred queue usage command | Minor | client | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6307](https://issues.apache.org/jira/browse/MAPREDUCE-6307) | Remove property mapreduce.tasktracker.taskmemorymanager.monitoringinterval | Minor | . | Akira Ajisaka | J.Andreina |
+| [YARN-3348](https://issues.apache.org/jira/browse/YARN-3348) | Add a 'yarn top' tool to help understand cluster usage | Major | resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-3469](https://issues.apache.org/jira/browse/YARN-3469) | ZKRMStateStore: Avoid setting watches that are not required | Minor | . | Jun Gong | Jun Gong |
+| [HDFS-8117](https://issues.apache.org/jira/browse/HDFS-8117) | More accurate verification in SimulatedFSDataset: replace DEFAULT\_DATABYTE with patterned data | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-8144](https://issues.apache.org/jira/browse/HDFS-8144) | Split TestLazyPersistFiles into multiple tests | Major | test | Arpit Agarwal | Arpit Agarwal |
+| [YARN-3404](https://issues.apache.org/jira/browse/YARN-3404) | View the queue name to YARN Application page | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [YARN-3451](https://issues.apache.org/jira/browse/YARN-3451) | Add start time and Elapsed in ApplicationAttemptReport and display the same in RMAttemptBlock WebUI | Major | api, webapp | Rohith Sharma K S | Rohith Sharma K S |
+| [HDFS-7863](https://issues.apache.org/jira/browse/HDFS-7863) | Missing description of some methods and parameters in javadoc of FSDirDeleteOp | Minor | . | Yongjun Zhang | Brahma Reddy Battula |
+| [HDFS-8152](https://issues.apache.org/jira/browse/HDFS-8152) | Refactoring of lazy persist storage cases | Major | test | Arpit Agarwal | Arpit Agarwal |
+| [HDFS-8133](https://issues.apache.org/jira/browse/HDFS-8133) | Improve readability of deleted block check | Major | namenode | Daryn Sharp | Daryn Sharp |
+| [HADOOP-11812](https://issues.apache.org/jira/browse/HADOOP-11812) | Implement listLocatedStatus for ViewFileSystem to speed up split calculation | Blocker | fs | Gera Shegalov | Gera Shegalov |
+| [MAPREDUCE-6297](https://issues.apache.org/jira/browse/MAPREDUCE-6297) | Task Id of the failed task in diagnostics should link to the task page | Minor | jobhistoryserver | Siqi Li | Siqi Li |
+| [HADOOP-11827](https://issues.apache.org/jira/browse/HADOOP-11827) | Speed-up distcp buildListing() using threadpool | Major | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [YARN-3494](https://issues.apache.org/jira/browse/YARN-3494) | Expose AM resource limit and usage in QueueMetrics | Major | . | Jian He | Rohith Sharma K S |
+| [YARN-3503](https://issues.apache.org/jira/browse/YARN-3503) | Expose disk utilization percentage and bad local and log dir counts on NM via JMX | Major | nodemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-3410](https://issues.apache.org/jira/browse/YARN-3410) | YARN admin should be able to remove individual application records from RMStateStore | Critical | resourcemanager, yarn | Wangda Tan | Rohith Sharma K S |
+| [HDFS-8215](https://issues.apache.org/jira/browse/HDFS-8215) | Refactor NamenodeFsck#check method | Minor | namenode | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-3511](https://issues.apache.org/jira/browse/YARN-3511) | Add errors and warnings page to ATS | Major | timelineserver | Varun Vasudev | Varun Vasudev |
+| [HDFS-8176](https://issues.apache.org/jira/browse/HDFS-8176) | Record from/to snapshots in audit log for snapshot diff report | Minor | snapshots | J.Andreina | J.Andreina |
+| [YARN-3406](https://issues.apache.org/jira/browse/YARN-3406) | Display count of running containers in the RM's Web UI | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-11357](https://issues.apache.org/jira/browse/HADOOP-11357) | Print information of the build enviornment in test-patch.sh | Minor | scripts | Haohui Mai | Allen Wittenauer |
+| [HADOOP-11857](https://issues.apache.org/jira/browse/HADOOP-11857) | Fix CommandFormat#commandFormat java doc annotation | Trivial | . | Xiaoyu Yao | J.Andreina |
+| [MAPREDUCE-6057](https://issues.apache.org/jira/browse/MAPREDUCE-6057) | Remove obsolete entries from mapred-default.xml | Minor | . | Ray Chiang | Ray Chiang |
+| [HDFS-8204](https://issues.apache.org/jira/browse/HDFS-8204) | Mover/Balancer should not schedule two replicas to the same DN | Minor | balancer & mover | Walter Su | Walter Su |
+| [HDFS-8280](https://issues.apache.org/jira/browse/HDFS-8280) | Code Cleanup in DFSInputStream | Minor | . | Jing Zhao | Jing Zhao |
+| [HDFS-8283](https://issues.apache.org/jira/browse/HDFS-8283) | DataStreamer cleanup and some minor improvement | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-5574](https://issues.apache.org/jira/browse/HDFS-5574) | Remove buffer copy in BlockReader.skip | Trivial | . | Binglin Chang | Binglin Chang |
+| [HDFS-7770](https://issues.apache.org/jira/browse/HDFS-7770) | Need document for storage type label of data node storage locations under dfs.datanode.data.dir | Major | documentation | Xiaoyu Yao | Xiaoyu Yao |
+| [HDFS-8200](https://issues.apache.org/jira/browse/HDFS-8200) | Refactor FSDirStatAndListingOp | Major | . | Haohui Mai | Haohui Mai |
+| [YARN-3363](https://issues.apache.org/jira/browse/YARN-3363) | add localization and container launch time to ContainerMetrics at NM to show these timing information for each active container. | Major | nodemanager | zhihai xu | zhihai xu |
+| [HDFS-7397](https://issues.apache.org/jira/browse/HDFS-7397) | Add more detail to the documentation for the conf key "dfs.client.read.shortcircuit.streams.cache.size" | Minor | hdfs-client | Tsz Wo Nicholas Sze | Brahma Reddy Battula |
+| [YARN-2980](https://issues.apache.org/jira/browse/YARN-2980) | Move health check script related functionality to hadoop-common | Blocker | . | Ming Ma | Varun Saxena |
+| [HADOOP-11911](https://issues.apache.org/jira/browse/HADOOP-11911) | test-patch should allow configuration of default branch | Minor | . | Sean Busbey | Sean Busbey |
+| [HDFS-7758](https://issues.apache.org/jira/browse/HDFS-7758) | Retire FsDatasetSpi#getVolumes() and use FsDatasetSpi#getVolumeRefs() instead | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6192](https://issues.apache.org/jira/browse/MAPREDUCE-6192) | Create unit test to automatically compare MR related classes and mapred-default.xml | Minor | . | Ray Chiang | Ray Chiang |
+| [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | releasedocmaker.py should use today's date instead of unreleased | Minor | build | Allen Wittenauer | Darrell Taylor |
+| [YARN-3491](https://issues.apache.org/jira/browse/YARN-3491) | PublicLocalizer#addResource is too slow. | Critical | nodemanager | zhihai xu | zhihai xu |
+| [MAPREDUCE-6279](https://issues.apache.org/jira/browse/MAPREDUCE-6279) | AM should explicity exit JVM after all services have stopped | Major | . | Jason Lowe | Eric Payne |
+| [HDFS-8108](https://issues.apache.org/jira/browse/HDFS-8108) | Fsck should provide the info on mandatory option to be used along with "-blocks , -locations and -racks" | Trivial | documentation | J.Andreina | J.Andreina |
+| [HDFS-8207](https://issues.apache.org/jira/browse/HDFS-8207) | Improper log message when blockreport interval compared with initial delay | Minor | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6079](https://issues.apache.org/jira/browse/MAPREDUCE-6079) | Rename JobImpl#username to reporterUserName | Major | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [HDFS-8209](https://issues.apache.org/jira/browse/HDFS-8209) | Support different number of datanode directories in MiniDFSCluster. | Minor | test | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [YARN-3169](https://issues.apache.org/jira/browse/YARN-3169) | Drop YARN's overview document | Major | documentation | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-2784](https://issues.apache.org/jira/browse/YARN-2784) | Make POM project names consistent | Minor | build | Rohith Sharma K S | Rohith Sharma K S |
+| [MAPREDUCE-5981](https://issues.apache.org/jira/browse/MAPREDUCE-5981) | Log levels of certain MR logs can be changed to DEBUG | Major | . | Varun Saxena | Varun Saxena |
+| [HADOOP-6842](https://issues.apache.org/jira/browse/HADOOP-6842) | "hadoop fs -text" does not give a useful text representation of MapWritable objects | Major | io | Steven K. Wong | Akira Ajisaka |
+| [YARN-20](https://issues.apache.org/jira/browse/YARN-20) | More information for "yarn.resourcemanager.webapp.address" in yarn-default.xml | Trivial | documentation, resourcemanager | Nemon Lou | Bartosz \u0141ugowski |
+| [HDFS-5640](https://issues.apache.org/jira/browse/HDFS-5640) | Add snapshot methods to FileContext. | Major | hdfs-client, snapshots | Chris Nauroth | Rakesh R |
+| [HDFS-8284](https://issues.apache.org/jira/browse/HDFS-8284) | Update documentation about how to use HTrace with HDFS | Major | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-7433](https://issues.apache.org/jira/browse/HDFS-7433) | Optimize performance of DatanodeManager's node map | Critical | namenode | Daryn Sharp | Daryn Sharp |
+| [HADOOP-11590](https://issues.apache.org/jira/browse/HADOOP-11590) | Update sbin commands and documentation to use new --slaves option | Blocker | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-5248](https://issues.apache.org/jira/browse/MAPREDUCE-5248) | Let NNBenchWithoutMR specify the replication factor for its test | Minor | client, test | Erik Paulson | Erik Paulson |
+| [HADOOP-9737](https://issues.apache.org/jira/browse/HADOOP-9737) | JarFinder#getJar should delete the jar file upon destruction of the JVM | Major | util | Esteban Gutierrez | Jean-Baptiste Onofr� |
+| [YARN-1050](https://issues.apache.org/jira/browse/YARN-1050) | Document the Fair Scheduler REST API | Major | documentation, fairscheduler | Sandy Ryza | Kenji Kikushima |
+| [YARN-3271](https://issues.apache.org/jira/browse/YARN-3271) | FairScheduler: Move tests related to max-runnable-apps from TestFairScheduler to TestAppRunnability | Major | . | Karthik Kambatla | nijel |
+| [YARN-2206](https://issues.apache.org/jira/browse/YARN-2206) | Update document for applications REST API response examples | Minor | documentation | Kenji Kikushima | Brahma Reddy Battula |
+| [HDFS-6757](https://issues.apache.org/jira/browse/HDFS-6757) | Simplify lease manager with INodeID | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-8327](https://issues.apache.org/jira/browse/HDFS-8327) | Simplify quota calculations for snapshots and truncate | Major | . | Haohui Mai | Haohui Mai |
+| [YARN-1287](https://issues.apache.org/jira/browse/YARN-1287) | Consolidate MockClocks | Major | . | Sandy Ryza | Sebastian Wong |
+| [HDFS-8357](https://issues.apache.org/jira/browse/HDFS-8357) | Consolidate parameters of INode.CleanSubtree() into a parameter objects. | Major | . | Haohui Mai | Li Lu |
+| [HADOOP-11950](https://issues.apache.org/jira/browse/HADOOP-11950) | Add cli option to test-patch to set the project-under-test | Minor | . | Sean Busbey | Sean Busbey |
+| [HADOOP-11948](https://issues.apache.org/jira/browse/HADOOP-11948) | test-patch's issue matching regex should be configurable. | Major | . | Sean Busbey | Sean Busbey |
+| [MAPREDUCE-5465](https://issues.apache.org/jira/browse/MAPREDUCE-5465) | Tasks are often killed before they exit on their own | Major | mr-am, mrv2 | Radim Kolar | Ming Ma |
+| [YARN-3513](https://issues.apache.org/jira/browse/YARN-3513) | Remove unused variables in ContainersMonitorImpl and add debug log for overall resource usage by all containers | Trivial | nodemanager | Naganarasimha G R | Naganarasimha G R |
+| [HDFS-8255](https://issues.apache.org/jira/browse/HDFS-8255) | Rename getBlockReplication to getPreferredBlockReplication | Major | . | Zhe Zhang | Zhe Zhang |
+| [YARN-3613](https://issues.apache.org/jira/browse/YARN-3613) | TestContainerManagerSecurity should init and start Yarn cluster in setup instead of individual methods | Minor | test | Karthik Kambatla | nijel |
+| [HDFS-6184](https://issues.apache.org/jira/browse/HDFS-6184) | Capture NN's thread dump when it fails over | Major | namenode | Ming Ma | Ming Ma |
+| [YARN-3539](https://issues.apache.org/jira/browse/YARN-3539) | Compatibility doc to state that ATS v1 is a stable REST API | Major | documentation | Steve Loughran | Steve Loughran |
+| [HADOOP-9723](https://issues.apache.org/jira/browse/HADOOP-9723) | Improve error message when hadoop archive output path already exists | Trivial | . | Stephen Chu | Yongjun Zhang |
+| [HADOOP-10993](https://issues.apache.org/jira/browse/HADOOP-10993) | Dump java command line to \*.out file | Minor | . | Allen Wittenauer | Kengo Seki |
+| [HADOOP-11713](https://issues.apache.org/jira/browse/HADOOP-11713) | ViewFileSystem should support snapshot methods. | Major | fs | Chris Nauroth | Rakesh R |
+| [HDFS-8350](https://issues.apache.org/jira/browse/HDFS-8350) | Remove old webhdfs.xml and other outdated documentation stuff | Major | documentation | Akira Ajisaka | Brahma Reddy Battula |
+| [HADOOP-11960](https://issues.apache.org/jira/browse/HADOOP-11960) | Enable Azure-Storage Client Side logging. | Major | tools | Dushyanth | Dushyanth |
+| [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | Allow selectively audit logging ops | Major | . | Kihwal Lee | Chen He |
+| [HDFS-8397](https://issues.apache.org/jira/browse/HDFS-8397) | Refactor the error handling code in DataStreamer | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-8394](https://issues.apache.org/jira/browse/HDFS-8394) | Move getAdditionalBlock() and related functionalities into a separate class | Major | . | Haohui Mai | Haohui Mai |
+| [HADOOP-11939](https://issues.apache.org/jira/browse/HADOOP-11939) | Deprecate DistCpV1 and Logalyzer | Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-4185](https://issues.apache.org/jira/browse/HDFS-4185) | Add a metric for number of active leases | Major | namenode | Kihwal Lee | Rakesh R |
+| [HADOOP-1540](https://issues.apache.org/jira/browse/HADOOP-1540) | Support file exclusion list in distcp | Minor | util | Senthil Subramanian | Rich Haase |
+| [HADOOP-11103](https://issues.apache.org/jira/browse/HADOOP-11103) | Clean up RemoteException | Trivial | ipc | Sean Busbey | Sean Busbey |
+| [HDFS-8131](https://issues.apache.org/jira/browse/HDFS-8131) | Implement a space balanced block placement policy | Minor | namenode | Liu Shaohui | Liu Shaohui |
+| [HADOOP-11970](https://issues.apache.org/jira/browse/HADOOP-11970) | Replace uses of ThreadLocal\<Random\> with JDK7 ThreadLocalRandom | Major | . | Sean Busbey | Sean Busbey |
+| [HADOOP-11995](https://issues.apache.org/jira/browse/HADOOP-11995) | Make jetty version configurable from the maven command line | Trivial | build, ld | sriharsha devineni | sriharsha devineni |
+| [HDFS-4383](https://issues.apache.org/jira/browse/HDFS-4383) | Document the lease limits | Minor | . | Eli Collins | Arshad Mohammad |
+| [HADOOP-10366](https://issues.apache.org/jira/browse/HADOOP-10366) | Add whitespaces between the classes for values in core-default.xml to fit better in browser | Minor | documentation | Chengwei Yang | Kanaka Kumar Avvaru |
+| [HADOOP-11594](https://issues.apache.org/jira/browse/HADOOP-11594) | Improve the readability of site index of documentation | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-160](https://issues.apache.org/jira/browse/YARN-160) | nodemanagers should obtain cpu/memory values from underlying OS | Major | nodemanager | Alejandro Abdelnur | Varun Vasudev |
+| [HADOOP-11242](https://issues.apache.org/jira/browse/HADOOP-11242) | Record the time of calling in tracing span of IPC server | Minor | ipc | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-5033](https://issues.apache.org/jira/browse/HDFS-5033) | Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source | Minor | . | Karthik Kambatla | Darrell Taylor |
+| [YARN-3722](https://issues.apache.org/jira/browse/YARN-3722) | Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils | Minor | test | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-11142](https://issues.apache.org/jira/browse/HADOOP-11142) | Remove hdfs dfs reference from file system shell documentation | Minor | documentation | Jonathan Allen | Kengo Seki |
+| [HADOOP-11894](https://issues.apache.org/jira/browse/HADOOP-11894) | Bump the version of Apache HTrace to 3.2.0-incubating | Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-3489](https://issues.apache.org/jira/browse/YARN-3489) | RMServerUtils.validateResourceRequests should only obtain queue info once | Major | resourcemanager | Jason Lowe | Varun Saxena |
+| [HDFS-8443](https://issues.apache.org/jira/browse/HDFS-8443) | Document dfs.namenode.service.handler.count in hdfs-site.xml | Major | documentation | Akira Ajisaka | J.Andreina |
+| [YARN-3547](https://issues.apache.org/jira/browse/YARN-3547) | FairScheduler: Apps that have no resource demand should not participate scheduling | Major | fairscheduler | Xianyin Xin | Xianyin Xin |
+| [YARN-3713](https://issues.apache.org/jira/browse/YARN-3713) | Remove duplicate function call storeContainerDiagnostics in ContainerDiagnosticsUpdateTransition | Minor | nodemanager | zhihai xu | zhihai xu |
+| [HADOOP-12043](https://issues.apache.org/jira/browse/HADOOP-12043) | Display warning if defaultFs is not set when running fs commands. | Minor | fs | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [YARN-3467](https://issues.apache.org/jira/browse/YARN-3467) | Expose allocatedMB, allocatedVCores, and runningContainers metrics on running Applications in RM Web UI | Minor | webapp, yarn | Anthony Rojas | Anubhav Dhoot |
+| [HDFS-8490](https://issues.apache.org/jira/browse/HDFS-8490) | Typo in trace enabled log in ExceptionHandler of WebHDFS | Trivial | webhdfs | Jakob Homan | Archana T |
+| [HDFS-8521](https://issues.apache.org/jira/browse/HDFS-8521) | Add @VisibleForTesting annotation to {{BlockPoolSlice#selectReplicaToDelete}} | Trivial | . | Colin P. McCabe | Colin P. McCabe |
+| [MAPREDUCE-6174](https://issues.apache.org/jira/browse/MAPREDUCE-6174) | Combine common stream code into parent class for InMemoryMapOutput and OnDiskMapOutput. | Major | mrv2 | Eric Payne | Eric Payne |
+| [HDFS-8532](https://issues.apache.org/jira/browse/HDFS-8532) | Make the visibility of DFSOutputStream#streamer member variable to private | Trivial | . | Rakesh R | Rakesh R |
+| [HDFS-8535](https://issues.apache.org/jira/browse/HDFS-8535) | Clarify that dfs usage in dfsadmin -report output includes all block replicas. | Minor | documentation | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6383](https://issues.apache.org/jira/browse/MAPREDUCE-6383) | Pi job (QuasiMonteCarlo) should not try to read the results file if its job fails | Major | examples | Harsh J | Harsh J |
+| [YARN-3259](https://issues.apache.org/jira/browse/YARN-3259) | FairScheduler: Trigger fairShare updates on node events | Major | fairscheduler | Anubhav Dhoot | Anubhav Dhoot |
+| [HADOOP-12059](https://issues.apache.org/jira/browse/HADOOP-12059) | S3Credentials should support use of CredentialProvider | Major | fs/s3 | Sean Busbey | Sean Busbey |
+| [MAPREDUCE-6354](https://issues.apache.org/jira/browse/MAPREDUCE-6354) | ShuffleHandler should be able to log shuffle connections | Major | . | Chang Li | Chang Li |
+| [HADOOP-12055](https://issues.apache.org/jira/browse/HADOOP-12055) | Deprecate usage of NativeIO#link | Major | native | Andrew Wang | Andrew Wang |
+| [HDFS-8432](https://issues.apache.org/jira/browse/HDFS-8432) | Introduce a minimum compatible layout version to allow downgrade in more rolling upgrade use cases. | Major | namenode, rolling upgrades | Chris Nauroth | Chris Nauroth |
+| [HDFS-8116](https://issues.apache.org/jira/browse/HDFS-8116) | Cleanup uncessary if LOG.isDebugEnabled() from RollingWindowManager | Trivial | namenode | Xiaoyu Yao | Brahma Reddy Battula |
+| [YARN-2716](https://issues.apache.org/jira/browse/YARN-2716) | Refactor ZKRMStateStore retry code with Apache Curator | Major | . | Jian He | Karthik Kambatla |
+| [HDFS-8553](https://issues.apache.org/jira/browse/HDFS-8553) | Document hdfs class path options | Major | documentation | Xiaoyu Yao | Brahma Reddy Battula |
+| [YARN-3786](https://issues.apache.org/jira/browse/YARN-3786) | Document yarn class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6392](https://issues.apache.org/jira/browse/MAPREDUCE-6392) | Document mapred class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8549](https://issues.apache.org/jira/browse/HDFS-8549) | Abort the balancer if an upgrade is in progress | Major | balancer & mover | Andrew Wang | Andrew Wang |
+| [HDFS-8573](https://issues.apache.org/jira/browse/HDFS-8573) | Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-11971](https://issues.apache.org/jira/browse/HADOOP-11971) | Move test utilities for tracing from hadoop-hdfs to hadoop-common | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-8361](https://issues.apache.org/jira/browse/HDFS-8361) | Choose SSD over DISK in block placement | Major | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [YARN-3789](https://issues.apache.org/jira/browse/YARN-3789) | Improve logs for LeafQueue#activateApplications() | Minor | resourcemanager | Bibin A Chundatt | Bibin A Chundatt |
+| [HDFS-8606](https://issues.apache.org/jira/browse/HDFS-8606) | Cleanup DFSOutputStream by removing unwanted changes | Minor | hdfs-client | Rakesh R | Rakesh R |
+| [YARN-3148](https://issues.apache.org/jira/browse/YARN-3148) | Allow CORS related headers to passthrough in WebAppProxyServlet | Major | . | Prakash Ramachandran | Varun Saxena |
+| [HDFS-8589](https://issues.apache.org/jira/browse/HDFS-8589) | Fix unused imports in BPServiceActor and BlockReportLeaseManager | Trivial | . | Colin P. McCabe | Colin P. McCabe |
+| [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | Allow appending to existing SequenceFiles | Major | io | Stephen Rose | Kanaka Kumar Avvaru |
+| [HDFS-8605](https://issues.apache.org/jira/browse/HDFS-8605) | Merge Refactor of DFSOutputStream from HDFS-7285 branch | Major | . | Vinayakumar B | Vinayakumar B |
+| [MAPREDUCE-6395](https://issues.apache.org/jira/browse/MAPREDUCE-6395) | Improve the commit failure messages in MRAppMaster recovery | Major | applicationmaster | Gera Shegalov | Brahma Reddy Battula |
+| [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings | Minor | datanode, hdfs-client | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6316](https://issues.apache.org/jira/browse/MAPREDUCE-6316) | Task Attempt List entries should link to the task overview | Major | . | Siqi Li | Siqi Li |
+| [MAPREDUCE-6305](https://issues.apache.org/jira/browse/MAPREDUCE-6305) | AM/Task log page should be able to link back to the job | Major | . | Siqi Li | Siqi Li |
+| [YARN-3834](https://issues.apache.org/jira/browse/YARN-3834) | Scrub debug logging of tokens during resource localization. | Major | nodemanager | Chris Nauroth | Chris Nauroth |
+| [MAPREDUCE-6408](https://issues.apache.org/jira/browse/MAPREDUCE-6408) | Queue name and user name should be printed on the job page | Major | applicationmaster | Siqi Li | Siqi Li |
+| [HDFS-8639](https://issues.apache.org/jira/browse/HDFS-8639) | Option for HTTP port of NameNode by MiniDFSClusterManager | Minor | test | Kai Sasaki | Kai Sasaki |
+| [YARN-3360](https://issues.apache.org/jira/browse/YARN-3360) | Add JMX metrics to TimelineDataManager | Major | timelineserver | Jason Lowe | Jason Lowe |
+| [HADOOP-12049](https://issues.apache.org/jira/browse/HADOOP-12049) | Control http authentication cookie persistence via configuration | Major | security | Benoy Antony | Huizhi Lu |
+| [HDFS-8462](https://issues.apache.org/jira/browse/HDFS-8462) | Implement GETXATTRS and LISTXATTRS operations for WebImageViewer | Major | . | Akira Ajisaka | Jagadesh Kiran N |
+| [HDFS-8640](https://issues.apache.org/jira/browse/HDFS-8640) | Make reserved RBW space visible through JMX | Major | . | Kanaka Kumar Avvaru | Kanaka Kumar Avvaru |
+| [HDFS-8546](https://issues.apache.org/jira/browse/HDFS-8546) | Use try with resources in DataStorage and Storage | Minor | datanode | Andrew Wang | Andrew Wang |
+| [HADOOP-11203](https://issues.apache.org/jira/browse/HADOOP-11203) | Allow ditscp to accept bandwitdh in fraction MegaBytes | Major | tools/distcp | Raju Bairishetti | Raju Bairishetti |
+| [HADOOP-11807](https://issues.apache.org/jira/browse/HADOOP-11807) | add a lint mode to releasedocmaker | Minor | build, documentation, yetus | Allen Wittenauer | ramtin |
+| [HDFS-8653](https://issues.apache.org/jira/browse/HDFS-8653) | Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-8659](https://issues.apache.org/jira/browse/HDFS-8659) | Block scanner INFO message is spamming logs | Major | datanode | Yongjun Zhang | Yongjun Zhang |
+| [HADOOP-12149](https://issues.apache.org/jira/browse/HADOOP-12149) | copy all of test-patch BINDIR prior to re-exec | Blocker | yetus | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6384](https://issues.apache.org/jira/browse/MAPREDUCE-6384) | Add the last reporting reducer info for too many fetch failure diagnostics | Major | . | Chang Li | Chang Li |
+| [HADOOP-12158](https://issues.apache.org/jira/browse/HADOOP-12158) | Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed | Trivial | test | Andrew Wang | Andrew Wang |
+| [HADOOP-12172](https://issues.apache.org/jira/browse/HADOOP-12172) | FsShell mkdir -p makes an unnecessary check for the existence of the parent. | Minor | fs | Chris Nauroth | Chris Nauroth |
+| [HDFS-8703](https://issues.apache.org/jira/browse/HDFS-8703) | Merge refactor of DFSInputStream from ErasureCoding branch | Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-8709](https://issues.apache.org/jira/browse/HDFS-8709) | Clarify automatic sync in FSEditLog#logEdit | Minor | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12045](https://issues.apache.org/jira/browse/HADOOP-12045) | Enable LocalFileSystem#setTimes to change atime | Minor | fs | Kazuho Fujii | Kazuho Fujii |
+| [HADOOP-12185](https://issues.apache.org/jira/browse/HADOOP-12185) | NetworkTopology is not efficient adding/getting/removing nodes | Major | . | Inigo Goiri | Inigo Goiri |
+| [HADOOP-12135](https://issues.apache.org/jira/browse/HADOOP-12135) | cleanup releasedocmaker | Major | yetus | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12195](https://issues.apache.org/jira/browse/HADOOP-12195) | Add annotation to package-info.java file to workaround MCOMPILER-205 | Trivial | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12193](https://issues.apache.org/jira/browse/HADOOP-12193) | Rename Touchz.java to Touch.java | Trivial | . | Andrew Wang | Andrew Wang |
+| [HDFS-8711](https://issues.apache.org/jira/browse/HDFS-8711) | setSpaceQuota command should print the available storage type when input storage type is wrong | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-8712](https://issues.apache.org/jira/browse/HDFS-8712) | Remove "public" and "abstract" modifiers in FsVolumeSpi and FsDatasetSpi | Trivial | . | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-12194](https://issues.apache.org/jira/browse/HADOOP-12194) | Support for incremental generation in the protoc plugin | Major | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12180](https://issues.apache.org/jira/browse/HADOOP-12180) | Move ResourceCalculatorPlugin from YARN to Common | Major | util | Chris Douglas | Chris Douglas |
+| [HADOOP-12210](https://issues.apache.org/jira/browse/HADOOP-12210) | Collect network usage on the node | Major | . | Robert Grandl | Robert Grandl |
+| [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | Document missing properties in yarn-default.xml | Major | documentation | Ray Chiang | Ray Chiang |
+| [YARN-3381](https://issues.apache.org/jira/browse/YARN-3381) | Fix typo InvalidStateTransitonException | Minor | api | Xiaoshuang LU | Brahma Reddy Battula |
+| [HADOOP-12211](https://issues.apache.org/jira/browse/HADOOP-12211) | Collect disks usages on the node | Major | . | Robert Grandl | Robert Grandl |
+| [HDFS-8722](https://issues.apache.org/jira/browse/HDFS-8722) | Optimize datanode writes for small writes and flushes | Critical | . | Kihwal Lee | Kihwal Lee |
+| [HADOOP-12232](https://issues.apache.org/jira/browse/HADOOP-12232) | Upgrade Tomcat dependency to 6.0.44. | Major | build | Chris Nauroth | Chris Nauroth |
+| [YARN-3170](https://issues.apache.org/jira/browse/YARN-3170) | YARN architecture document needs updating | Major | documentation | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-3174](https://issues.apache.org/jira/browse/YARN-3174) | Consolidate the NodeManager and NodeManagerRestart documentation into one | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [HDFS-7314](https://issues.apache.org/jira/browse/HDFS-7314) | When the DFSClient lease cannot be renewed, abort open-for-write files rather than the entire DFSClient | Major | . | Ming Ma | Ming Ma |
+| [HADOOP-10979](https://issues.apache.org/jira/browse/HADOOP-10979) | Auto-entries in hadoop\_usage | Minor | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11893](https://issues.apache.org/jira/browse/HADOOP-11893) | Mark org.apache.hadoop.security.token.Token as @InterfaceAudience.Public | Major | security | Steve Loughran | Brahma Reddy Battula |
+| [HADOOP-12081](https://issues.apache.org/jira/browse/HADOOP-12081) | Fix UserGroupInformation.java to support 64-bit zLinux | Major | security | Adam Roberts | Akira Ajisaka |
+| [HADOOP-12214](https://issues.apache.org/jira/browse/HADOOP-12214) | Parse 'HadoopArchive' commandline using cli Options. | Minor | . | Vinayakumar B | Vinayakumar B |
+| [YARN-3261](https://issues.apache.org/jira/browse/YARN-3261) | rewrite resourcemanager restart doc to remove roadmap bits | Major | documentation | Allen Wittenauer | Gururaj Shetty |
+| [YARN-2921](https://issues.apache.org/jira/browse/YARN-2921) | Fix MockRM/MockAM#waitForState sleep too long | Major | test | Karthik Kambatla | Tsuyoshi Ozawa |
+| [HADOOP-12161](https://issues.apache.org/jira/browse/HADOOP-12161) | Add getStoragePolicy API to the FileSystem interface | Major | . | Arpit Agarwal | Brahma Reddy Battula |
+| [HADOOP-12189](https://issues.apache.org/jira/browse/HADOOP-12189) | Improve CallQueueManager#swapQueue to make queue elements drop nearly impossible. | Major | ipc, test | zhihai xu | zhihai xu |
+| [HADOOP-12009](https://issues.apache.org/jira/browse/HADOOP-12009) | Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus | Minor | documentation, fs, test | Jakob Homan | J.Andreina |
+| [HADOOP-12259](https://issues.apache.org/jira/browse/HADOOP-12259) | Utility to Dynamic port allocation | Major | test, util | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8735](https://issues.apache.org/jira/browse/HDFS-8735) | Inotify : All events classes should implement toString() API. | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-7858](https://issues.apache.org/jira/browse/HDFS-7858) | Improve HA Namenode Failover detection on the client | Major | hdfs-client | Arun Suresh | Arun Suresh |
+| [HDFS-8180](https://issues.apache.org/jira/browse/HDFS-8180) | AbstractFileSystem Implementation for WebHdfs | Major | webhdfs | Santhosh G Nayak | Santhosh G Nayak |
+| [HDFS-8811](https://issues.apache.org/jira/browse/HDFS-8811) | Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java | Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-8822](https://issues.apache.org/jira/browse/HDFS-8822) | Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies | Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-3950](https://issues.apache.org/jira/browse/YARN-3950) | Add unique YARN\_SHELL\_ID environment variable to DistributedShell | Major | applications/distributed-shell | Robert Kanter | Robert Kanter |
+| [YARN-2768](https://issues.apache.org/jira/browse/YARN-2768) | Avoid cloning Resource in FSAppAttempt#updateDemand | Minor | fairscheduler | Hong Zhiguo | Hong Zhiguo |
+| [HDFS-8816](https://issues.apache.org/jira/browse/HDFS-8816) | Improve visualization for the Datanode tab in the NN UI | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-8821](https://issues.apache.org/jira/browse/HDFS-8821) | Explain message "Operation category X is not supported in state standby" | Minor | . | Gautam Gopalakrishnan | Gautam Gopalakrishnan |
+| [HADOOP-12271](https://issues.apache.org/jira/browse/HADOOP-12271) | Hadoop Jar Error Should Be More Explanatory | Minor | . | Jesse Anderson | Josh Elser |
+| [HADOOP-12249](https://issues.apache.org/jira/browse/HADOOP-12249) | pull argument parsing into a function | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12183](https://issues.apache.org/jira/browse/HADOOP-12183) | Annotate the HTrace span created by FsShell with the command-line arguments passed by the user | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-3978](https://issues.apache.org/jira/browse/YARN-3978) | Configurably turn off the saving of container info in Generic AHS | Major | timelineserver, yarn | Eric Payne | Eric Payne |
+| [YARN-3965](https://issues.apache.org/jira/browse/YARN-3965) | Ad
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/43] hadoop git commit: HDFS-10847. Complete the document for
FileDistribution processor in OfflineImageViewer. Contributed by Yiqun Lin.
Posted by ae...@apache.org.
HDFS-10847. Complete the document for FileDistribution processor in OfflineImageViewer. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20a20c2f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20a20c2f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20a20c2f
Branch: refs/heads/HDFS-7240
Commit: 20a20c2f6e1b3b8aa6a58a824ad3aadc349dc761
Parents: 2d1bf53
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Sep 9 01:34:34 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Sep 9 01:34:34 2016 +0900
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 3 +++
.../hadoop-hdfs/src/site/markdown/HdfsImageViewer.md | 12 +++++++++---
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a20c2f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 4075878..83035bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -261,6 +261,9 @@ Usage: `hdfs oiv_legacy [OPTIONS] -i INPUT_FILE -o OUTPUT_FILE`
| COMMAND\_OPTION | Description |
|:---- |:---- |
| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Valid options are Ls (default), XML, Delimited, Indented, and FileDistribution. |
+| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
+| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
+| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
| `-skipBlocks` | Do not enumerate individual blocks within files. This may save processing time and outfile file space on namespaces with very large files. The Ls processor reads the blocks to correctly determine file sizes and ignores this option. |
| `-printToScreen` | Pipe output of processor to console as well as specified file. On extremely large namespaces, this may increase processing time by an order of magnitude. |
| `-delimiter` *arg* | When used in conjunction with the Delimited processor, replaces the default tab delimiter with the string specified by *arg*. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20a20c2f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index f991b44..f55c9fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -50,10 +50,13 @@ The Offline Image Viewer provides several output processors:
..., s[n-1], maxSize], and the processor calculates how many files
in the system fall into each segment [s[i-1], s[i]). Note that
files larger than maxSize always fall into the very last segment.
- The output file is formatted as a tab separated two column table:
- Size and NumFiles. Where Size represents the start of the segment,
+ By default, the output file is formatted as a tab separated two column
+ table: Size and NumFiles. Where Size represents the start of the segment,
and numFiles is the number of files form the image which size falls
- in this segment.
+ in this segment. By specifying the option -format, the output file will be
+ formatted in a human-readable fashion rather than a number of bytes that
+ showed in Size column. In addition, the Size column will be changed to the
+ Size Range column.
4. Delimited (experimental): Generate a text file with all of the elements
common to both inodes and inodes-under-construction, separated by a
@@ -182,6 +185,9 @@ Due to the internal layout changes introduced by the ProtocolBuffer-based fsimag
| `-i`\|`--inputFile` *input file* | Specify the input fsimage file to process. Required. |
| `-o`\|`--outputFile` *output file* | Specify the output filename, if the specified output processor generates one. If the specified file already exists, it is silently overwritten. Required. |
| `-p`\|`--processor` *processor* | Specify the image processor to apply against the image file. Valid options are Ls (default), XML, Delimited, Indented, and FileDistribution. |
+| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
+| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
+| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
| `-skipBlocks` | Do not enumerate individual blocks within files. This may save processing time and outfile file space on namespaces with very large files. The Ls processor reads the blocks to correctly determine file sizes and ignores this option. |
| `-printToScreen` | Pipe output of processor to console as well as specified file. On extremely large namespaces, this may increase processing time by an order of magnitude. |
| `-delimiter` *arg* | When used in conjunction with the Delimited processor, replaces the default tab delimiter with the string specified by *arg*. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/43] hadoop git commit: YARN-5264. Store all queue-specific
information in FSQueue. (Yufei Gu via kasha)
Posted by ae...@apache.org.
YARN-5264. Store all queue-specific information in FSQueue. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6ea9be5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6ea9be5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6ea9be5
Branch: refs/heads/HDFS-7240
Commit: f6ea9be5473ab66798b0536317d2f32c5348eb57
Parents: cbd909c
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Fri Sep 2 14:56:29 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Fri Sep 2 14:56:29 2016 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationConfiguration.java | 83 ++++++++++++--------
.../scheduler/fair/FSLeafQueue.java | 17 ++--
.../scheduler/fair/FSParentQueue.java | 8 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 71 ++++++++++++++---
.../scheduler/fair/FairScheduler.java | 7 +-
.../scheduler/fair/MaxRunningAppsEnforcer.java | 11 +--
.../scheduler/fair/QueueManager.java | 45 ++---------
.../webapp/dao/FairSchedulerQueueInfo.java | 2 +-
.../scheduler/fair/TestAppRunnability.java | 6 +-
.../scheduler/fair/TestFSLeafQueue.java | 10 +--
.../scheduler/fair/TestFairScheduler.java | 11 ++-
.../fair/TestMaxRunningAppsEnforcer.java | 20 ++---
.../scheduler/fair/TestQueueManager.java | 17 ++--
13 files changed, 160 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index 5cf110f..c771887 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -22,6 +22,8 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
public class AllocationConfiguration extends ReservationSchedulerConfiguration {
+ private static final Log LOG = LogFactory.getLog(FSQueue.class.getName());
private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
private static final ResourceCalculator RESOURCE_CALCULATOR =
@@ -242,26 +245,24 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return !nonPreemptableQueues.contains(queueName);
}
- public ResourceWeights getQueueWeight(String queue) {
+ private ResourceWeights getQueueWeight(String queue) {
ResourceWeights weight = queueWeights.get(queue);
return (weight == null) ? ResourceWeights.NEUTRAL : weight;
}
- public void setQueueWeight(String queue, ResourceWeights weight) {
- queueWeights.put(queue, weight);
- }
-
public int getUserMaxApps(String user) {
Integer maxApps = userMaxApps.get(user);
return (maxApps == null) ? userMaxAppsDefault : maxApps;
}
- public int getQueueMaxApps(String queue) {
+ @VisibleForTesting
+ int getQueueMaxApps(String queue) {
Integer maxApps = queueMaxApps.get(queue);
return (maxApps == null) ? queueMaxAppsDefault : maxApps;
}
-
- public float getQueueMaxAMShare(String queue) {
+
+ @VisibleForTesting
+ float getQueueMaxAMShare(String queue) {
Float maxAMShare = queueMaxAMShares.get(queue);
return (maxAMShare == null) ? queueMaxAMShareDefault : maxAMShare;
}
@@ -273,29 +274,21 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
* @return the min allocation on this queue or {@link Resources#none}
* if not set
*/
- public Resource getMinResources(String queue) {
+ @VisibleForTesting
+ Resource getMinResources(String queue) {
Resource minQueueResource = minQueueResources.get(queue);
return (minQueueResource == null) ? Resources.none() : minQueueResource;
}
/**
- * Set the maximum resource allocation for the given queue.
- *
- * @param queue the target queue
- * @param maxResource the maximum resource allocation
- */
- void setMaxResources(String queue, Resource maxResource) {
- maxQueueResources.put(queue, maxResource);
- }
-
- /**
* Get the maximum resource allocation for the given queue. If the max in not
* set, return the larger of the min and the default max.
*
* @param queue the target queue's name
* @return the max allocation on this queue
*/
- public Resource getMaxResources(String queue) {
+ @VisibleForTesting
+ Resource getMaxResources(String queue) {
Resource maxQueueResource = maxQueueResources.get(queue);
if (maxQueueResource == null) {
Resource minQueueResource = minQueueResources.get(queue);
@@ -317,21 +310,11 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
* @param queue the target queue's name
* @return the max allocation on this queue or null if not set
*/
- public Resource getMaxChildResources(String queue) {
+ @VisibleForTesting
+ Resource getMaxChildResources(String queue) {
return maxChildQueueResources.get(queue);
}
- /**
- * Set the maximum resource allocation for the children of the given queue.
- * Use of this method is primarily intended for testing purposes.
- *
- * @param queue the target queue
- * @param maxResource the maximum resource allocation
- */
- void setMaxChildResources(String queue, Resource maxResource) {
- maxChildQueueResources.put(queue, maxResource);
- }
-
public boolean hasAccess(String queueName, QueueACL acl,
UserGroupInformation user) {
int lastPeriodIndex = queueName.length();
@@ -346,8 +329,9 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return false;
}
-
- public SchedulingPolicy getSchedulingPolicy(String queueName) {
+
+ @VisibleForTesting
+ SchedulingPolicy getSchedulingPolicy(String queueName) {
SchedulingPolicy policy = schedulingPolicies.get(queueName);
return (policy == null) ? defaultSchedulingPolicy : policy;
}
@@ -423,4 +407,35 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
public void setAverageCapacity(int avgCapacity) {
globalReservationQueueConfig.setAverageCapacity(avgCapacity);
}
+
+ /**
+ * Initialize a {@link FSQueue} with queue-specific properties and its
+ * metrics.
+ * @param queue the FSQueue needed to be initialized
+ * @param scheduler the scheduler which the queue belonged to
+ */
+ public void initFSQueue(FSQueue queue, FairScheduler scheduler){
+ // Set queue-specific properties.
+ String name = queue.getName();
+ queue.setWeights(getQueueWeight(name));
+ queue.setMinShare(getMinResources(name));
+ queue.setMaxShare(getMaxResources(name));
+ queue.setMaxRunningApps(getQueueMaxApps(name));
+ queue.setMaxAMShare(getQueueMaxAMShare(name));
+ queue.setMaxChildQueueResource(getMaxChildResources(name));
+ try {
+ SchedulingPolicy policy = getSchedulingPolicy(name);
+ policy.initialize(scheduler.getClusterResource());
+ queue.setPolicy(policy);
+ } catch (AllocationConfigurationException ex) {
+ LOG.warn("Failed to set the scheduling policy "
+ + getDefaultSchedulingPolicy(), ex);
+ }
+
+ // Set queue metrics.
+ queue.getMetrics().setMinShare(getMinResources(name));
+ queue.getMetrics().setMaxShare(getMaxResources(name));
+ queue.getMetrics().setMaxApps(getQueueMaxApps(name));
+ queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 7c9b11e..a6adb47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -264,29 +264,27 @@ public class FSLeafQueue extends FSQueue {
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
- Resource maxRes = scheduler.getAllocationConfiguration()
- .getMaxResources(getName());
demand = Resources.createResource(0);
readLock.lock();
try {
for (FSAppAttempt sched : runnableApps) {
- if (Resources.equals(demand, maxRes)) {
+ if (Resources.equals(demand, maxShare)) {
break;
}
- updateDemandForApp(sched, maxRes);
+ updateDemandForApp(sched, maxShare);
}
for (FSAppAttempt sched : nonRunnableApps) {
- if (Resources.equals(demand, maxRes)) {
+ if (Resources.equals(demand, maxShare)) {
break;
}
- updateDemandForApp(sched, maxRes);
+ updateDemandForApp(sched, maxShare);
}
} finally {
readLock.unlock();
}
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand
- + "; the max is " + maxRes);
+ + "; the max is " + maxShare);
LOG.debug("The updated fairshare for " + getName() + " is "
+ getFairShare());
}
@@ -486,8 +484,6 @@ public class FSLeafQueue extends FSQueue {
* @return true if this queue can run
*/
public boolean canRunAppAM(Resource amResource) {
- float maxAMShare =
- scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName());
if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
return true;
}
@@ -543,8 +539,7 @@ public class FSLeafQueue extends FSQueue {
* @param weight queue weight
*/
public void setWeights(float weight) {
- scheduler.getAllocationConfiguration().setQueueWeight(getName(),
- new ResourceWeights(weight));
+ this.weights = new ResourceWeights(weight);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 035c60c..e58c3f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -152,8 +152,6 @@ public class FSParentQueue extends FSQueue {
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
- Resource maxRes = scheduler.getAllocationConfiguration()
- .getMaxResources(getName());
writeLock.lock();
try {
demand = Resources.createResource(0);
@@ -166,8 +164,8 @@ public class FSParentQueue extends FSQueue {
" now " + demand);
}
demand = Resources.add(demand, toAdd);
- demand = Resources.componentwiseMin(demand, maxRes);
- if (Resources.equals(demand, maxRes)) {
+ demand = Resources.componentwiseMin(demand, maxShare);
+ if (Resources.equals(demand, maxShare)) {
break;
}
}
@@ -176,7 +174,7 @@ public class FSParentQueue extends FSQueue {
}
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand +
- "; the max is " + maxRes);
+ "; the max is " + maxShare);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 25554dd..5fa2ee1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -59,6 +59,15 @@ public abstract class FSQueue implements Queue, Schedulable {
protected SchedulingPolicy policy = SchedulingPolicy.DEFAULT_POLICY;
+ protected ResourceWeights weights;
+ protected Resource minShare;
+ protected Resource maxShare;
+ protected int maxRunningApps;
+ protected Resource maxChildQueueResource;
+
+ // maxAMShare is a value between 0 and 1.
+ protected float maxAMShare;
+
private long fairSharePreemptionTimeout = Long.MAX_VALUE;
private long minSharePreemptionTimeout = Long.MAX_VALUE;
private float fairSharePreemptionThreshold = 0.5f;
@@ -68,15 +77,20 @@ public abstract class FSQueue implements Queue, Schedulable {
this.name = name;
this.scheduler = scheduler;
this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
- metrics.setMinShare(getMinShare());
- metrics.setMaxShare(getMaxShare());
+ this.parent = parent;
+ }
+ /**
+ * Initialize a queue by setting its queue-specific properties and its
+ * metrics.
+ * This function is invoked when a new queue is created or reloading the
+ * allocation configuration.
+ */
+ public void init() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- metrics.setMaxApps(allocConf.getQueueMaxApps(name));
- metrics.setSchedulingPolicy(allocConf.getSchedulingPolicy(name).getName());
- this.parent = parent;
+ allocConf.initFSQueue(this, scheduler);
}
-
+
public String getName() {
return name;
}
@@ -103,19 +117,51 @@ public abstract class FSQueue implements Queue, Schedulable {
public abstract void setPolicy(SchedulingPolicy policy)
throws AllocationConfigurationException;
+ public void setWeights(ResourceWeights weights){
+ this.weights = weights;
+ }
+
@Override
public ResourceWeights getWeights() {
- return scheduler.getAllocationConfiguration().getQueueWeight(getName());
+ return weights;
}
-
+
+ public void setMinShare(Resource minShare){
+ this.minShare = minShare;
+ }
+
@Override
public Resource getMinShare() {
- return scheduler.getAllocationConfiguration().getMinResources(getName());
+ return minShare;
}
-
+
+ public void setMaxShare(Resource maxShare){
+ this.maxShare = maxShare;
+ }
+
@Override
public Resource getMaxShare() {
- return scheduler.getAllocationConfiguration().getMaxResources(getName());
+ return maxShare;
+ }
+
+ public void setMaxChildQueueResource(Resource maxChildShare){
+ this.maxChildQueueResource = maxChildShare;
+ }
+
+ public Resource getMaxChildQueueResource() {
+ return maxChildQueueResource;
+ }
+
+ public void setMaxRunningApps(int maxRunningApps){
+ this.maxRunningApps = maxRunningApps;
+ }
+
+ public int getMaxRunningApps() {
+ return maxRunningApps;
+ }
+
+ public void setMaxAMShare(float maxAMShare){
+ this.maxAMShare = maxAMShare;
}
@Override
@@ -302,8 +348,7 @@ public abstract class FSQueue implements Queue, Schedulable {
* @return true if check passes (can assign) or false otherwise
*/
protected boolean assignContainerPreCheck(FSSchedulerNode node) {
- if (!Resources.fitsIn(getResourceUsage(),
- scheduler.getAllocationConfiguration().getMaxResources(getName()))
+ if (!Resources.fitsIn(getResourceUsage(), maxShare)
|| node.getReservedContainer() != null) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 140b4f7..ac8dbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1599,11 +1599,10 @@ public class FairScheduler extends
if ((queue.getParent() != null) &&
!configuredLeafQueues.contains(queue.getName()) &&
!configuredParentQueues.contains(queue.getName())) {
- Resource max =
- allocConf.getMaxChildResources(queue.getParent().getName());
+ Resource max = queue.getParent().getMaxChildQueueResource();
if (max != null) {
- allocConf.setMaxResources(queue.getName(), max);
+ queue.setMaxShare(max);
}
}
}
@@ -1665,7 +1664,7 @@ public class FairScheduler extends
FSQueue cur = targetQueue;
while (cur != lowestCommonAncestor) {
// maxRunningApps
- if (cur.getNumRunnableApps() == allocConf.getQueueMaxApps(cur.getQueueName())) {
+ if (cur.getNumRunnableApps() == cur.getMaxRunningApps()) {
throw new YarnException("Moving app attempt " + appAttId + " to queue "
+ queueName + " would violate queue maxRunningApps constraints on"
+ " queue " + cur.getQueueName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
index f750438..8592fa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
@@ -66,8 +66,7 @@ public class MaxRunningAppsEnforcer {
}
// Check queue and all parent queues
while (queue != null) {
- int queueMaxApps = allocConf.getQueueMaxApps(queue.getName());
- if (queue.getNumRunnableApps() >= queueMaxApps) {
+ if (queue.getNumRunnableApps() >= queue.getMaxRunningApps()) {
return false;
}
queue = queue.getParent();
@@ -143,11 +142,10 @@ public class MaxRunningAppsEnforcer {
// Thus we find the ancestor queue highest in the tree for which the app
// that was at its maxRunningApps before the removal.
FSQueue highestQueueWithAppsNowRunnable = (queue.getNumRunnableApps() ==
- allocConf.getQueueMaxApps(queue.getName()) - 1) ? queue : null;
+ queue.getMaxRunningApps() - 1) ? queue : null;
FSParentQueue parent = queue.getParent();
while (parent != null) {
- if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent
- .getName()) - 1) {
+ if (parent.getNumRunnableApps() == parent.getMaxRunningApps() - 1) {
highestQueueWithAppsNowRunnable = parent;
}
parent = parent.getParent();
@@ -265,8 +263,7 @@ public class MaxRunningAppsEnforcer {
*/
private void gatherPossiblyRunnableAppLists(FSQueue queue,
List<List<FSAppAttempt>> appLists) {
- if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration()
- .getQueueMaxApps(queue.getName())) {
+ if (queue.getNumRunnableApps() < queue.getMaxRunningApps()) {
if (queue instanceof FSLeafQueue) {
appLists.add(
((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 964c72b..bca0ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -73,8 +73,9 @@ public class QueueManager {
public void initialize(Configuration conf) throws IOException,
SAXException, AllocationConfigurationException, ParserConfigurationException {
rootQueue = new FSParentQueue("root", scheduler, null);
+ rootQueue.init();
queues.put(rootQueue.getName(), rootQueue);
-
+
// Create the default queue
getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true);
}
@@ -174,7 +175,7 @@ public class QueueManager {
}
return queue;
}
-
+
/**
* Create a leaf or parent queue based on what is specified in
* {@code queueType} and place it in the tree. Create any parents that don't
@@ -273,31 +274,14 @@ public class QueueManager {
// Only create a leaf queue at the very end
if (!i.hasNext() && (queueType != FSQueueType.PARENT)) {
FSLeafQueue leafQueue = new FSLeafQueue(queueName, scheduler, parent);
-
- try {
- leafQueue.setPolicy(queueConf.getDefaultSchedulingPolicy());
- } catch (AllocationConfigurationException ex) {
- LOG.warn("Failed to set default scheduling policy "
- + queueConf.getDefaultSchedulingPolicy()
- + " on new leaf queue.", ex);
- }
-
leafQueues.add(leafQueue);
queue = leafQueue;
} else {
newParent = new FSParentQueue(queueName, scheduler, parent);
-
- try {
- newParent.setPolicy(queueConf.getDefaultSchedulingPolicy());
- } catch (AllocationConfigurationException ex) {
- LOG.warn("Failed to set default scheduling policy "
- + queueConf.getDefaultSchedulingPolicy()
- + " on new parent queue.", ex);
- }
-
queue = newParent;
}
+ queue.init();
parent.addChildQueue(queue);
setChildResourceLimits(parent, queue, queueConf);
queues.put(queue.getName(), queue);
@@ -331,10 +315,10 @@ public class QueueManager {
!configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) {
// For ad hoc queues, set their max reource allocations based on
// their parents' default child settings.
- Resource maxChild = queueConf.getMaxChildResources(parent.getName());
+ Resource maxChild = parent.getMaxChildQueueResource();
if (maxChild != null) {
- queueConf.setMaxResources(child.getName(), maxChild);
+ child.setMaxShare(maxChild);
}
}
}
@@ -515,22 +499,7 @@ public class QueueManager {
rootQueue.recomputeSteadyShares();
for (FSQueue queue : queues.values()) {
- // Update queue metrics
- FSQueueMetrics queueMetrics = queue.getMetrics();
- queueMetrics.setMinShare(queue.getMinShare());
- queueMetrics.setMaxShare(queue.getMaxShare());
- // Set scheduling policies and update queue metrics
- try {
- SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName());
- policy.initialize(scheduler.getClusterResource());
- queue.setPolicy(policy);
-
- queueMetrics.setMaxApps(queueConf.getQueueMaxApps(queue.getName()));
- queueMetrics.setSchedulingPolicy(policy.getName());
- } catch (AllocationConfigurationException ex) {
- LOG.warn("Cannot apply configured scheduling policy to queue "
- + queue.getName(), ex);
- }
+ queue.init();
}
// Update steady fair shares for all queues
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index a09b92f..64e573b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -101,7 +101,7 @@ public class FairSchedulerQueueInfo {
fractionMemMinShare = (float)minResources.getMemorySize() / clusterResources.getMemorySize();
fractionMemMaxShare = (float)maxResources.getMemorySize() / clusterResources.getMemorySize();
- maxApps = allocConf.getQueueMaxApps(queueName);
+ maxApps = queue.getMaxRunningApps();
pendingContainers = queue.getMetrics().getPendingContainers();
allocatedContainers = queue.getMetrics().getAllocatedContainers();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
index 7ca9606..f581935 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
@@ -247,8 +247,8 @@ public class TestAppRunnability extends FairSchedulerTestBase {
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
+ oldQueue.setMaxRunningApps(0);
+ targetQueue.setMaxRunningApps(0);
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
@@ -265,7 +265,7 @@ public class TestAppRunnability extends FairSchedulerTestBase {
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
+ oldQueue.setMaxRunningApps(0);
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index ad4e2e4..0a2ce81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileWriter;
@@ -80,16 +79,11 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
resourceManager = new MockRM(conf);
resourceManager.start();
scheduler = (FairScheduler) resourceManager.getResourceScheduler();
- scheduler.allocConf = mock(AllocationConfiguration.class);
String queueName = "root.queue1";
- when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource);
- when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none());
- when(scheduler.allocConf.getQueueMaxApps(queueName)).
- thenReturn(Integer.MAX_VALUE);
- when(scheduler.allocConf.getSchedulingPolicy(queueName))
- .thenReturn(SchedulingPolicy.DEFAULT_POLICY);
FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);
+ schedulable.init();
+ schedulable.setMaxShare(maxResource);
assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE);
assertEquals(schedulable.getMetrics().getSchedulingPolicy(),
SchedulingPolicy.DEFAULT_POLICY.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index c348572..98af8b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4348,9 +4348,9 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.reinitialize(conf, resourceManager.getRMContext());
QueueManager queueMgr = scheduler.getQueueManager();
- queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
-
+ FSQueue queue2 = queueMgr.getLeafQueue("queue2", true);
+ queue2.setMaxRunningApps(0);
+
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
@@ -4365,9 +4365,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
- queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2",
- Resource.newInstance(1024, 1));
+ FSQueue queue2 = queueMgr.getLeafQueue("queue2", true);
+ queue2.setMaxShare(Resource.newInstance(1024, 1));
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
index 6cca19a..3f17081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
@@ -37,7 +37,6 @@ import org.junit.Test;
public class TestMaxRunningAppsEnforcer {
private QueueManager queueManager;
- private Map<String, Integer> queueMaxApps;
private Map<String, Integer> userMaxApps;
private MaxRunningAppsEnforcer maxAppsEnforcer;
private int appNum;
@@ -59,7 +58,6 @@ public class TestMaxRunningAppsEnforcer {
queueManager = new QueueManager(scheduler);
queueManager.initialize(conf);
- queueMaxApps = allocConf.queueMaxApps;
userMaxApps = allocConf.userMaxApps;
maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
appNum = 0;
@@ -90,11 +88,12 @@ public class TestMaxRunningAppsEnforcer {
@Test
public void testRemoveDoesNotEnableAnyApp() {
+ FSParentQueue root = queueManager.getRootQueue();
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue2", true);
- queueMaxApps.put("root", 2);
- queueMaxApps.put("root.queue1", 1);
- queueMaxApps.put("root.queue2", 1);
+ root.setMaxRunningApps(2);
+ leaf1.setMaxRunningApps(1);
+ leaf2.setMaxRunningApps(1);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
@@ -111,7 +110,8 @@ public class TestMaxRunningAppsEnforcer {
public void testRemoveEnablesAppOnCousinQueue() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
- queueMaxApps.put("root.queue1", 2);
+ FSParentQueue queue1 = queueManager.getParentQueue("root.queue1", true);
+ queue1.setMaxRunningApps(2);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
@@ -128,7 +128,7 @@ public class TestMaxRunningAppsEnforcer {
public void testRemoveEnablesOneByQueueOneByUser() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true);
- queueMaxApps.put("root.queue1.leaf1", 2);
+ leaf1.setMaxRunningApps(2);
userMaxApps.put("user1", 1);
FSAppAttempt app1 = addApp(leaf1, "user1");
addApp(leaf1, "user2");
@@ -148,7 +148,8 @@ public class TestMaxRunningAppsEnforcer {
public void testRemoveEnablingOrderedByStartTime() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
- queueMaxApps.put("root.queue1", 2);
+ FSParentQueue queue1 = queueManager.getParentQueue("root.queue1", true);
+ queue1.setMaxRunningApps(2);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
@@ -168,7 +169,8 @@ public class TestMaxRunningAppsEnforcer {
public void testMultipleAppsWaitingOnCousinQueue() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
- queueMaxApps.put("root.queue1", 2);
+ FSParentQueue queue1 = queueManager.getParentQueue("root.queue1", true);
+ queue1.setMaxRunningApps(2);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index a9c3788..05ab11c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -45,8 +45,6 @@ public class TestQueueManager {
// Set up some queues to test default child max resource inheritance
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test");
- allocConf.setMaxChildResources("root.test",
- Resources.createResource(8192, 256));
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA");
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB");
@@ -207,16 +205,17 @@ public class TestQueueManager {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
queueManager.updateAllocationConfiguration(allocConf);
+ queueManager.getQueue("root.test").setMaxChildQueueResource(
+ Resources.createResource(8192, 256));
FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF);
-
assertNotNull("Leaf queue root.test.childC was not created",
queueManager.getLeafQueue("root.test.childC", false));
assertEquals("createQueue() returned wrong queue",
"root.test.childC", q1.getName());
assertEquals("Max resources for root.queue1 were not inherited from "
+ "parent's max child resources", Resources.createResource(8192, 256),
- allocConf.getMaxResources("root.test.childC"));
+ q1.getMaxShare());
FSQueue q2 = queueManager.createQueue("root.test.childD",
FSQueueType.PARENT);
@@ -228,7 +227,7 @@ public class TestQueueManager {
assertEquals("Max resources for root.test.childD were not inherited "
+ "from parent's max child resources",
Resources.createResource(8192, 256),
- allocConf.getMaxResources("root.test.childD"));
+ q2.getMaxShare());
// Check that the childA and childB queues weren't impacted
// by the child defaults
@@ -236,12 +235,12 @@ public class TestQueueManager {
queueManager.getLeafQueue("root.test.childA", false));
assertEquals("Max resources for root.test.childA were inherited from "
+ "parent's max child resources", Resources.unbounded(),
- allocConf.getMaxResources("root.test.childA"));
+ queueManager.getLeafQueue("root.test.childA", false).getMaxShare());
assertNotNull("Leaf queue root.test.childB was not created during setup",
queueManager.getParentQueue("root.test.childB", false));
assertEquals("Max resources for root.test.childB were inherited from "
+ "parent's max child resources", Resources.unbounded(),
- allocConf.getMaxResources("root.test.childB"));
+ queueManager.getParentQueue("root.test.childB", false).getMaxShare());
}
/**
@@ -260,11 +259,11 @@ public class TestQueueManager {
// Min default is 0,0
assertEquals("Min resources were not set to default",
Resources.createResource(0, 0),
- allocConf.getMinResources("root.queue1"));
+ q1.getMinShare());
// Max default is unbounded
assertEquals("Max resources were not set to default", Resources.unbounded(),
- allocConf.getMaxResources("root.queue1"));
+ q1.getMaxShare());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/43] hadoop git commit: HDFS-6962. ACL inheritance conflicts with
umaskmode. Contributed by Chris Nauroth.
Posted by ae...@apache.org.
HDFS-6962. ACL inheritance conflicts with umaskmode. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0d5382f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0d5382f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0d5382f
Branch: refs/heads/HDFS-7240
Commit: f0d5382ff3e31a47d13e4cb6c3a244cca82b17ce
Parents: d37dc5d
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Sep 6 11:02:39 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Sep 6 11:02:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileContext.java | 8 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 7 +-
.../hadoop/fs/permission/FsCreateModes.java | 101 ++
.../hadoop/fs/permission/FsPermission.java | 14 +
.../hadoop/fs/permission/package-info.java | 25 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +-
.../ClientNamenodeProtocolTranslatorPB.java | 14 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hdfs/web/resources/PermissionParam.java | 9 +-
.../web/resources/UnmaskedPermissionParam.java | 51 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
...tNamenodeProtocolServerSideTranslatorPB.java | 16 +-
.../datanode/web/webhdfs/ParameterParser.java | 7 +
.../datanode/web/webhdfs/WebHdfsHandler.java | 6 +-
.../hadoop/hdfs/server/namenode/AclStorage.java | 7 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 3 +-
.../hdfs/server/namenode/FSDirSymlinkOp.java | 3 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 5 +-
.../hdfs/server/namenode/FSDirectory.java | 72 +-
.../web/resources/NamenodeWebHdfsMethods.java | 49 +-
.../src/main/resources/hdfs-default.xml | 14 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 6 +
.../java/org/apache/hadoop/cli/TestAclCLI.java | 6 +-
.../cli/TestAclCLIWithPosixAclInheritance.java | 45 +
.../hdfs/server/namenode/FSAclBaseTest.java | 91 ++
.../server/namenode/TestGetBlockLocations.java | 2 +-
.../snapshot/TestRenameWithSnapshots.java | 4 +-
.../testAclCLIWithPosixAclInheritance.xml | 1075 ++++++++++++++++++
29 files changed, 1613 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index f235773..160a63d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
@@ -671,7 +672,7 @@ public class FileContext {
CreateOpts.Perms permOpt = CreateOpts.getOpt(CreateOpts.Perms.class, opts);
FsPermission permission = (permOpt != null) ? permOpt.getValue() :
FILE_DEFAULT_PERM;
- permission = permission.applyUMask(getUMask());
+ permission = FsCreateModes.applyUMask(permission, getUMask());
final CreateOpts[] updatedOpts =
CreateOpts.setOpt(CreateOpts.perms(permission), opts);
@@ -717,8 +718,9 @@ public class FileContext {
ParentNotDirectoryException, UnsupportedFileSystemException,
IOException {
final Path absDir = fixRelativePart(dir);
- final FsPermission absFerms = (permission == null ?
- FsPermission.getDirDefault() : permission).applyUMask(getUMask());
+ final FsPermission absFerms = FsCreateModes.applyUMask(
+ permission == null ?
+ FsPermission.getDirDefault() : permission, getUMask());
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c366729..9bde29d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.Text;
@@ -925,9 +926,9 @@ public abstract class FileSystem extends Configured implements Closeable {
long blockSize,
Progressable progress
) throws IOException {
- return this.create(f, FsPermission.getFileDefault().applyUMask(
- FsPermission.getUMask(getConf())), overwrite, bufferSize,
- replication, blockSize, progress);
+ return this.create(f, FsCreateModes.applyUMask(
+ FsPermission.getFileDefault(), FsPermission.getUMask(getConf())),
+ overwrite, bufferSize, replication, blockSize, progress);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
new file mode 100644
index 0000000..a1ed0d7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.permission;
+
+import java.text.MessageFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A class that stores both masked and unmasked create modes
+ * and is a drop-in replacement for masked permission.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class FsCreateModes extends FsPermission {
+ private final FsPermission unmasked;
+
+ /**
+ * Create from unmasked mode and umask.
+ *
+ * If the mode is already an FsCreateModes object, return it.
+ */
+ public static FsPermission applyUMask(FsPermission mode,
+ FsPermission umask) {
+ if (mode.getUnmasked() != null) {
+ return mode;
+ }
+ return create(mode.applyUMask(umask), mode);
+ }
+
+ /**
+ * Create from masked and unmasked modes.
+ */
+ public static FsCreateModes create(FsPermission masked,
+ FsPermission unmasked) {
+ assert masked.getUnmasked() == null;
+ assert unmasked.getUnmasked() == null;
+ return new FsCreateModes(masked, unmasked);
+ }
+
+ private FsCreateModes(FsPermission masked, FsPermission unmasked) {
+ super(masked);
+ this.unmasked = unmasked;
+ assert masked.getUnmasked() == null;
+ assert unmasked.getUnmasked() == null;
+ }
+
+ @Override
+ public FsPermission getMasked() {
+ return this;
+ }
+
+ @Override
+ public FsPermission getUnmasked() {
+ return unmasked;
+ }
+
+ @Override
+ public String toString() {
+ return MessageFormat.format("'{' masked: {0}, unmasked: {1} '}'",
+ super.toString(), getUnmasked());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ FsCreateModes that = (FsCreateModes) o;
+ return getUnmasked().equals(that.getUnmasked());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + getUnmasked().hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index 78255bbd..48a5b1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -138,6 +138,20 @@ public class FsPermission implements Writable {
}
/**
+ * Get masked permission if exists.
+ */
+ public FsPermission getMasked() {
+ return null;
+ }
+
+ /**
+ * Get unmasked permission if exists.
+ */
+ public FsPermission getUnmasked() {
+ return null;
+ }
+
+ /**
* Create and initialize a {@link FsPermission} from {@link DataInput}.
*/
public static FsPermission read(DataInput in) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
new file mode 100644
index 0000000..6cee4ee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package provides support for HDFS permission and ACL.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.permission;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ad86895..b0fcba2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -1160,14 +1161,14 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
if (permission == null) {
permission = FsPermission.getFileDefault();
}
- return permission.applyUMask(dfsClientConf.getUMask());
+ return FsCreateModes.applyUMask(permission, dfsClientConf.getUMask());
}
private FsPermission applyUMaskDir(FsPermission permission) {
if (permission == null) {
permission = FsPermission.getDirDefault();
}
- return permission.applyUMask(dfsClientConf.getUMask());
+ return FsCreateModes.applyUMask(permission, dfsClientConf.getUMask());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 57f8fd6..f73abfd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -294,6 +294,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setCreateParent(createParent)
.setReplication(replication)
.setBlockSize(blockSize);
+ FsPermission unmasked = masked.getUnmasked();
+ if (unmasked != null) {
+ builder.setUnmasked(PBHelperClient.convert(unmasked));
+ }
builder.addAllCryptoProtocolVersion(
PBHelperClient.convert(supportedVersions));
CreateRequestProto req = builder.build();
@@ -579,11 +583,15 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public boolean mkdirs(String src, FsPermission masked, boolean createParent)
throws IOException {
- MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
+ MkdirsRequestProto.Builder builder = MkdirsRequestProto.newBuilder()
.setSrc(src)
.setMasked(PBHelperClient.convert(masked))
- .setCreateParent(createParent).build();
-
+ .setCreateParent(createParent);
+ FsPermission unmasked = masked.getUnmasked();
+ if (unmasked != null) {
+ builder.setUnmasked(PBHelperClient.convert(unmasked));
+ }
+ MkdirsRequestProto req = builder.build();
try {
return rpcProxy.mkdirs(null, req).getResult();
} catch (ServiceException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 968eea3..5389a02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
@@ -973,7 +974,8 @@ public class WebHdfsFileSystem extends FileSystem
if (permission == null) {
permission = FsPermission.getDefault();
}
- return permission.applyUMask(FsPermission.getUMask(getConf()));
+ return FsCreateModes.applyUMask(permission,
+ FsPermission.getUMask(getConf()));
}
private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
@@ -1025,8 +1027,10 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.MKDIRS);
final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
+ final FsPermission modes = applyUMask(permission);
return new FsPathBooleanRunner(op, f,
- new PermissionParam(applyUMask(permission))
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked())
).run();
}
@@ -1313,9 +1317,11 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.CREATE);
+ final FsPermission modes = applyUMask(permission);
final HttpOpParam.Op op = PutOpParam.Op.CREATE;
return new FsPathOutputStreamRunner(op, f, bufferSize,
- new PermissionParam(applyUMask(permission)),
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked()),
new OverwriteParam(overwrite),
new BufferSizeParam(bufferSize),
new ReplicationParam(replication),
@@ -1331,9 +1337,11 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.CREATE_NON_RECURSIVE);
+ final FsPermission modes = applyUMask(permission);
final HttpOpParam.Op op = PutOpParam.Op.CREATE;
return new FsPathOutputStreamRunner(op, f, bufferSize,
- new PermissionParam(applyUMask(permission)),
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked()),
new CreateFlagParam(flag),
new CreateParentParam(false),
new BufferSizeParam(bufferSize),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
index 42ff69d..1598cbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
@@ -54,7 +54,7 @@ public class PermissionParam extends ShortParam {
* @param value the parameter value.
*/
public PermissionParam(final FsPermission value) {
- super(DOMAIN, value == null? null: value.toShort(), null, null);
+ this(DOMAIN, value == null ? null : value.toShort(), null, null);
}
/**
@@ -62,7 +62,12 @@ public class PermissionParam extends ShortParam {
* @param str a string representation of the parameter value.
*/
public PermissionParam(final String str) {
- super(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ this(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ }
+
+ PermissionParam(final Domain domain, final Short value, final Short min,
+ final Short max) {
+ super(domain, value, min, max);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
new file mode 100644
index 0000000..bbd2a5c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/**
+ * Unmasked permission parameter, use a Short to represent a FsPermission.
+ */
+public class UnmaskedPermissionParam extends PermissionParam {
+ /** Parameter name. */
+ public static final String NAME = "unmaskedpermission";
+
+ private static final Domain DOMAIN = new Domain(NAME, 8);
+
+ /**
+ * Constructor.
+ * @param value the parameter value.
+ */
+ public UnmaskedPermissionParam(final FsPermission value) {
+ super(DOMAIN, value == null ? null : value.toShort(), null, null);
+ }
+
+ /**
+ * Constructor.
+ * @param str a string representation of the parameter value.
+ */
+ public UnmaskedPermissionParam(final String str) {
+ super(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 83f5828..54bed32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -79,6 +79,7 @@ message CreateRequestProto {
required uint32 replication = 6; // Short: Only 16 bits used
required uint64 blockSize = 7;
repeated CryptoProtocolVersionProto cryptoProtocolVersion = 8;
+ optional FsPermissionProto unmasked = 9;
}
message CreateResponseProto {
@@ -264,6 +265,7 @@ message MkdirsRequestProto {
required string src = 1;
required FsPermissionProto masked = 2;
required bool createParent = 3;
+ optional FsPermissionProto unmasked = 4;
}
message MkdirsResponseProto {
required bool result = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f2ab321..7bdfd44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -259,6 +259,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = "supergroup";
public static final String DFS_NAMENODE_ACLS_ENABLED_KEY = "dfs.namenode.acls.enabled";
public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+ public static final String DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY =
+ "dfs.namenode.posix.acl.inheritance.enabled";
+ public static final boolean
+ DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = false;
public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled";
public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
public static final String DFS_ADMIN = "dfs.cluster.administrators";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 641cd20..3974956 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.permission.FsCreateModes;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
@@ -411,8 +413,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public CreateResponseProto create(RpcController controller,
CreateRequestProto req) throws ServiceException {
try {
+ FsPermission masked = req.hasUnmasked() ?
+ FsCreateModes.create(PBHelperClient.convert(req.getMasked()),
+ PBHelperClient.convert(req.getUnmasked())) :
+ PBHelperClient.convert(req.getMasked());
HdfsFileStatus result = server.create(req.getSrc(),
- PBHelperClient.convert(req.getMasked()), req.getClientName(),
+ masked, req.getClientName(),
PBHelperClient.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(),
(short) req.getReplication(), req.getBlockSize(),
PBHelperClient.convertCryptoProtocolVersions(
@@ -651,8 +657,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public MkdirsResponseProto mkdirs(RpcController controller,
MkdirsRequestProto req) throws ServiceException {
try {
- boolean result = server.mkdirs(req.getSrc(),
- PBHelperClient.convert(req.getMasked()), req.getCreateParent());
+ FsPermission masked = req.hasUnmasked() ?
+ FsCreateModes.create(PBHelperClient.convert(req.getMasked()),
+ PBHelperClient.convert(req.getUnmasked())) :
+ PBHelperClient.convert(req.getMasked());
+ boolean result = server.mkdirs(req.getSrc(), masked,
+ req.getCreateParent());
return MkdirsResponseProto.newBuilder().setResult(result).build();
} catch (IOException e) {
throw new ServiceException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index 3c7016e..bb28110 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.web.resources.OffsetParam;
import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
import org.apache.hadoop.hdfs.web.resources.PermissionParam;
import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
+import org.apache.hadoop.hdfs.web.resources.UnmaskedPermissionParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.token.Token;
@@ -108,6 +109,12 @@ class ParameterParser {
getFileFsPermission();
}
+ FsPermission unmaskedPermission() {
+ String value = param(UnmaskedPermissionParam.NAME);
+ return value == null ? null :
+ new UnmaskedPermissionParam(value).getFileFsPermission();
+ }
+
boolean overwrite() {
return new OverwriteParam(param(OverwriteParam.NAME)).getValue();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index d354612..20e0a2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -66,6 +66,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
@@ -187,7 +188,10 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
final int bufferSize = params.bufferSize();
final short replication = params.replication();
final long blockSize = params.blockSize();
- final FsPermission permission = params.permission();
+ final FsPermission unmaskedPermission = params.unmaskedPermission();
+ final FsPermission permission = unmaskedPermission == null ?
+ params.permission() :
+ FsCreateModes.create(params.permission(), unmaskedPermission);
final boolean createParent = params.createParent();
EnumSet<CreateFlag> flags = params.createFlag();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index abd3755..58dfe12 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -73,11 +73,11 @@ public final class AclStorage {
*
* @param child INode newly created child
*/
- public static void copyINodeDefaultAcl(INode child) {
+ public static boolean copyINodeDefaultAcl(INode child) {
INodeDirectory parent = child.getParent();
AclFeature parentAclFeature = parent.getAclFeature();
if (parentAclFeature == null || !(child.isFile() || child.isDirectory())) {
- return;
+ return false;
}
// Split parent's entries into access vs. default.
@@ -88,7 +88,7 @@ public final class AclStorage {
// The parent may have an access ACL but no default ACL. If so, exit.
if (parentDefaultEntries.isEmpty()) {
- return;
+ return false;
}
// Pre-allocate list size for access entries to copy from parent.
@@ -145,6 +145,7 @@ public final class AclStorage {
}
child.setPermission(newPerm);
+ return true;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index bf5ff00..2d1914f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -219,7 +219,8 @@ class FSDirMkdirOp {
final INodeDirectory dir = new INodeDirectory(inodeId, name, permission,
timestamp);
- INodesInPath iip = fsd.addLastINode(parent, dir, true);
+ INodesInPath iip =
+ fsd.addLastINode(parent, dir, permission.getPermission(), true);
if (iip != null && aclEntries != null) {
AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
index ec93952..6938a84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -88,7 +88,8 @@ class FSDirSymlinkOp {
final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
target);
symlink.setLocalName(localName);
- return fsd.addINode(iip, symlink) != null ? symlink : null;
+ return fsd.addINode(iip, symlink, perm.getPermission()) != null ?
+ symlink : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 077f04f..aa2be92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -496,7 +496,8 @@ class FSDirWriteFileOp {
replication, preferredBlockSize, storagePolicyId, ecPolicy != null);
}
newNode.setLocalName(localName);
- INodesInPath iip = fsd.addINode(existing, newNode);
+ INodesInPath iip = fsd.addINode(existing, newNode,
+ permissions.getPermission());
if (iip != null) {
if (aclEntries != null) {
AclStorage.updateINodeAcl(newNode, aclEntries, CURRENT_STATE_ID);
@@ -594,7 +595,7 @@ class FSDirWriteFileOp {
modTime, modTime, replication, preferredBlockSize, ecPolicy != null);
newNode.setLocalName(localName);
newNode.toUnderConstruction(clientName, clientMachine);
- newiip = fsd.addINode(existing, newNode);
+ newiip = fsd.addINode(existing, newNode, permissions.getPermission());
} finally {
fsd.writeUnlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 5dc5a03..340d491 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -168,6 +168,10 @@ public class FSDirectory implements Closeable {
* ACL-related operations.
*/
private final boolean aclsEnabled;
+ /**
+ * Support for POSIX ACL inheritance. Not final for testing purpose.
+ */
+ private boolean posixAclInheritanceEnabled;
private final boolean xattrsEnabled;
private final int xattrMaxSize;
@@ -251,6 +255,10 @@ public class FSDirectory implements Closeable {
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
LOG.info("ACLs enabled? " + aclsEnabled);
+ this.posixAclInheritanceEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT);
+ LOG.info("POSIX ACL inheritance enabled? " + posixAclInheritanceEnabled);
this.xattrsEnabled = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
@@ -450,6 +458,18 @@ public class FSDirectory implements Closeable {
boolean isAclsEnabled() {
return aclsEnabled;
}
+
+ @VisibleForTesting
+ public boolean isPosixAclInheritanceEnabled() {
+ return posixAclInheritanceEnabled;
+ }
+
+ @VisibleForTesting
+ public void setPosixAclInheritanceEnabled(
+ boolean posixAclInheritanceEnabled) {
+ this.posixAclInheritanceEnabled = posixAclInheritanceEnabled;
+ }
+
boolean isXattrsEnabled() {
return xattrsEnabled;
}
@@ -952,16 +972,18 @@ public class FSDirectory implements Closeable {
* Add the given child to the namespace.
* @param existing the INodesInPath containing all the ancestral INodes
* @param child the new INode to add
+ * @param modes create modes
* @return a new INodesInPath instance containing the new child INode. Null
* if the adding fails.
* @throws QuotaExceededException is thrown if it violates quota limit
*/
- INodesInPath addINode(INodesInPath existing, INode child)
+ INodesInPath addINode(INodesInPath existing, INode child,
+ FsPermission modes)
throws QuotaExceededException, UnresolvedLinkException {
cacheName(child);
writeLock();
try {
- return addLastINode(existing, child, true);
+ return addLastINode(existing, child, modes, true);
} finally {
writeUnlock();
}
@@ -1067,12 +1089,51 @@ public class FSDirectory implements Closeable {
}
/**
+ * Turn on HDFS-6962 POSIX ACL inheritance when the property
+ * {@link DFSConfigKeys#DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY} is
+ * true and a compatible client has sent both masked and unmasked create
+ * modes.
+ *
+ * @param child INode newly created child
+ * @param modes create modes
+ */
+ private void copyINodeDefaultAcl(INode child, FsPermission modes) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("child: {}, posixAclInheritanceEnabled: {}, modes: {}",
+ child, posixAclInheritanceEnabled, modes);
+ }
+
+ if (posixAclInheritanceEnabled && modes != null &&
+ modes.getUnmasked() != null) {
+ //
+ // HDFS-6962: POSIX ACL inheritance
+ //
+ child.setPermission(modes.getUnmasked());
+ if (!AclStorage.copyINodeDefaultAcl(child)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}: no parent default ACL to inherit", child);
+ }
+ child.setPermission(modes.getMasked());
+ }
+ } else {
+ //
+ // Old behavior before HDFS-6962
+ //
+ AclStorage.copyINodeDefaultAcl(child);
+ }
+ }
+
+ /**
* Add a child to the end of the path specified by INodesInPath.
+ * @param existing the INodesInPath containing all the ancestral INodes
+ * @param inode the new INode to add
+ * @param modes create modes
+ * @param checkQuota whether to check quota
* @return an INodesInPath instance containing the new INode
*/
@VisibleForTesting
public INodesInPath addLastINode(INodesInPath existing, INode inode,
- boolean checkQuota) throws QuotaExceededException {
+ FsPermission modes, boolean checkQuota) throws QuotaExceededException {
assert existing.getLastINode() != null &&
existing.getLastINode().isDirectory();
@@ -1119,7 +1180,7 @@ public class FSDirectory implements Closeable {
return null;
} else {
if (!isRename) {
- AclStorage.copyINodeDefaultAcl(inode);
+ copyINodeDefaultAcl(inode, modes);
}
addToInodeMap(inode);
}
@@ -1128,7 +1189,8 @@ public class FSDirectory implements Closeable {
INodesInPath addLastINodeNoQuotaCheck(INodesInPath existing, INode i) {
try {
- return addLastINode(existing, i, false);
+ // All callers do not have create modes to pass.
+ return addLastINode(existing, i, null, false);
} catch (QuotaExceededException e) {
NameNode.LOG.warn("FSDirectory.addChildNoQuotaCheck - unexpected", e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index e3a5bad..3ab0c67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -324,6 +326,9 @@ public class NamenodeWebHdfsMethods {
final GroupParam group,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
+ @QueryParam(UnmaskedPermissionParam.NAME)
+ @DefaultValue(UnmaskedPermissionParam.DEFAULT)
+ final UnmaskedPermissionParam unmaskedPermission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
@@ -362,11 +367,11 @@ public class NamenodeWebHdfsMethods {
final NoRedirectParam noredirect
) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
- owner, group, permission, overwrite, bufferSize, replication,
- blockSize, modificationTime, accessTime, renameOptions, createParent,
- delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
- createFlagParam, noredirect);
+ owner, group, permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime, renameOptions,
+ createParent, delegationTokenArgument, aclPermission, xattrName,
+ xattrValue, xattrSetFlag, snapshotName, oldSnapshotName,
+ excludeDatanodes, createFlagParam, noredirect);
}
/** Handle HTTP PUT request. */
@@ -393,6 +398,9 @@ public class NamenodeWebHdfsMethods {
final GroupParam group,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
+ @QueryParam(UnmaskedPermissionParam.NAME)
+ @DefaultValue(UnmaskedPermissionParam.DEFAULT)
+ final UnmaskedPermissionParam unmaskedPermission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
@@ -432,10 +440,11 @@ public class NamenodeWebHdfsMethods {
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
- group, permission, overwrite, bufferSize, replication, blockSize,
- modificationTime, accessTime, renameOptions, delegationTokenArgument,
- aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
- oldSnapshotName, excludeDatanodes, createFlagParam, noredirect);
+ group, permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime, renameOptions,
+ delegationTokenArgument, aclPermission, xattrName, xattrValue,
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ createFlagParam, noredirect);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -443,10 +452,11 @@ public class NamenodeWebHdfsMethods {
try {
return put(ugi, delegation, username, doAsUser,
path.getAbsolutePath(), op, destination, owner, group,
- permission, overwrite, bufferSize, replication, blockSize,
- modificationTime, accessTime, renameOptions, createParent,
- delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime,
+ renameOptions, createParent, delegationTokenArgument,
+ aclPermission, xattrName, xattrValue, xattrSetFlag,
+ snapshotName, oldSnapshotName, excludeDatanodes,
createFlagParam, noredirect);
} finally {
reset();
@@ -466,6 +476,7 @@ public class NamenodeWebHdfsMethods {
final OwnerParam owner,
final GroupParam group,
final PermissionParam permission,
+ final UnmaskedPermissionParam unmaskedPermission,
final OverwriteParam overwrite,
final BufferSizeParam bufferSize,
final ReplicationParam replication,
@@ -495,8 +506,9 @@ public class NamenodeWebHdfsMethods {
{
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser, fullpath, op.getValue(), -1L, blockSize.getValue(conf),
- exclDatanodes.getValue(), permission, overwrite, bufferSize,
- replication, blockSize, createParent, createFlagParam);
+ exclDatanodes.getValue(), permission, unmaskedPermission,
+ overwrite, bufferSize, replication, blockSize, createParent,
+ createFlagParam);
if(!noredirectParam.getValue()) {
return Response.temporaryRedirect(uri)
.type(MediaType.APPLICATION_OCTET_STREAM).build();
@@ -507,8 +519,11 @@ public class NamenodeWebHdfsMethods {
}
case MKDIRS:
{
- final boolean b = np.mkdirs(fullpath,
- permission.getDirFsPermission(), true);
+ FsPermission masked = unmaskedPermission.getValue() == null ?
+ permission.getDirFsPermission() :
+ FsCreateModes.create(permission.getDirFsPermission(),
+ unmaskedPermission.getDirFsPermission());
+ final boolean b = np.mkdirs(fullpath, masked, true);
final String js = JsonUtil.toJsonString("boolean", b);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4eca80c..d427f88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -457,7 +457,19 @@
</description>
</property>
-<property>
+ <property>
+ <name>dfs.namenode.posix.acl.inheritance.enabled</name>
+ <value>false</value>
+ <description>
+ Set to true to enable POSIX style ACL inheritance. When it is enabled
+ and the create request comes from a compatible client, the NameNode
+ will apply default ACLs from the parent directory to the create mode
+ and ignore the client umask. If no default ACL found, it will apply the
+ client umask.
+ </description>
+ </property>
+
+ <property>
<name>dfs.namenode.lazypersist.file.scrub.interval.sec</name>
<value>300</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
index ba027ac..7fd9b00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
@@ -333,4 +333,10 @@ Configuration Parameters
default, ACLs are disabled. When ACLs are disabled, the NameNode rejects
all attempts to set an ACL.
+* `dfs.namenode.posix.acl.inheritance.enabled`
+ Set to true to enable POSIX style ACL inheritance. Disabled by default.
+ When it is enabled and the create request comes from a compatible client,
+ the NameNode will apply default ACLs from the parent directory to
+ the create mode and ignore the client umask. If no default ACL is found,
+ it will apply the client umask.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
index 061d328..75111bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
@@ -32,11 +32,15 @@ public class TestAclCLI extends CLITestHelperDFS {
private String namenode = null;
private String username = null;
+ protected void initConf() {
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ }
+
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ initConf();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
fs = cluster.getFileSystem();
namenode = conf.get(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "file:///");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
new file mode 100644
index 0000000..ec31766
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cli;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY;
+
+import org.junit.Test;
+
+/**
+ * Test ACL CLI with POSIX ACL inheritance enabled.
+ */
+public class TestAclCLIWithPosixAclInheritance extends TestAclCLI {
+
+ @Override
+ protected void initConf() {
+ super.initConf();
+ conf.setBoolean(DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY, true);
+ }
+
+ @Override
+ protected String getTestFile() {
+ return "testAclCLIWithPosixAclInheritance.xml";
+ }
+
+ @Test
+ @Override
+ public void testAll() {
+ super.testAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
index 216147a..71614f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY;
import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
import static org.apache.hadoop.fs.permission.AclEntryScope.*;
import static org.apache.hadoop.fs.permission.AclEntryType.*;
@@ -889,6 +890,46 @@ public abstract class FSAclBaseTest {
}
@Test
+ public void testUMaskDefaultAclNewFile() throws Exception {
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, READ_WRITE),
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ String oldUMask = fs.getConf().get(FS_PERMISSIONS_UMASK_KEY);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, "027");
+
+ FSDirectory fsDirectory = cluster.getNamesystem().getFSDirectory();
+ boolean oldEnabled = fsDirectory.isPosixAclInheritanceEnabled();
+
+ try {
+ fsDirectory.setPosixAclInheritanceEnabled(false);
+ Path filePath = new Path(path, "file1");
+ fs.create(filePath).close();
+ AclStatus s = fs.getAclStatus(filePath);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_WRITE)}, returned);
+ assertPermission(filePath, (short) 010640);
+
+ fsDirectory.setPosixAclInheritanceEnabled(true);
+ Path file2Path = new Path(path, "file2");
+ fs.create(file2Path).close();
+ AclStatus s2 = fs.getAclStatus(file2Path);
+ AclEntry[] returned2 = s2.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_WRITE)}, returned2);
+ assertPermission(file2Path, (short) 010660);
+ } finally {
+ fsDirectory.setPosixAclInheritanceEnabled(oldEnabled);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, oldUMask);
+ }
+ }
+
+ @Test
public void testOnlyAccessAclNewFile() throws Exception {
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -943,6 +984,56 @@ public abstract class FSAclBaseTest {
}
@Test
+ public void testUMaskDefaultAclNewDir() throws Exception {
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ String oldUMask = fs.getConf().get(FS_PERMISSIONS_UMASK_KEY);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, "027");
+
+ FSDirectory fsDirectory = cluster.getNamesystem().getFSDirectory();
+ boolean oldEnabled = fsDirectory.isPosixAclInheritanceEnabled();
+
+ try {
+ fsDirectory.setPosixAclInheritanceEnabled(false);
+ Path dirPath = new Path(path, "dir1");
+ fs.mkdirs(dirPath);
+ AclStatus s = fs.getAclStatus(dirPath);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, ALL),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, NONE)}, returned);
+ assertPermission(dirPath, (short) 010750);
+
+ fsDirectory.setPosixAclInheritanceEnabled(true);
+ Path dir2Path = new Path(path, "dir2");
+ fs.mkdirs(dir2Path);
+ AclStatus s2 = fs.getAclStatus(dir2Path);
+ AclEntry[] returned2 = s2.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, ALL),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, NONE)}, returned2);
+ assertPermission(dir2Path, (short) 010770);
+ } finally {
+ fsDirectory.setPosixAclInheritanceEnabled(oldEnabled);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, oldUMask);
+ }
+ }
+
+ @Test
public void testOnlyAccessAclNewDir() throws Exception {
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
List<AclEntry> aclSpec = Lists.newArrayList(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
index eec5c98..0eb7132 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
@@ -127,7 +127,7 @@ public class TestGetBlockLocations {
MOCK_INODE_ID, FILE_NAME.getBytes(StandardCharsets.UTF_8),
perm, 1, 1, new BlockInfo[] {}, (short) 1,
DFS_BLOCK_SIZE_DEFAULT);
- fsn.getFSDirectory().addINode(iip, file);
+ fsn.getFSDirectory().addINode(iip, file, null);
return fsn;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index ad3a5a1..91eec78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -1570,7 +1570,9 @@ public class TestRenameWithSnapshots {
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new NSQuotaExceededException("fake exception")).when(fsdir2)
.addLastINode((INodesInPath) Mockito.anyObject(),
- (INode) Mockito.anyObject(), Mockito.anyBoolean());
+ (INode) Mockito.anyObject(),
+ (FsPermission) Mockito.anyObject(),
+ Mockito.anyBoolean());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/subdir2/foo.
// FSDirectory#verifyQuota4Rename will pass since the remaining quota is 2.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/43] hadoop git commit: HDFS-10553. DiskBalancer: Rename
Tools/DiskBalancer class to Tools/DiskBalancerCLI. Contributed by Manoj
Govindassamy.
Posted by ae...@apache.org.
HDFS-10553. DiskBalancer: Rename Tools/DiskBalancer class to Tools/DiskBalancerCLI. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35c5943b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35c5943b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35c5943b
Branch: refs/heads/HDFS-7240
Commit: 35c5943b8ba394191405555cdfc5e6127053ee97
Parents: b07c266
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Sep 8 19:26:56 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Sep 8 19:26:56 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/bin/hdfs | 2 +-
.../diskbalancer/command/CancelCommand.java | 23 +-
.../server/diskbalancer/command/Command.java | 6 +-
.../diskbalancer/command/ExecuteCommand.java | 15 +-
.../diskbalancer/command/HelpCommand.java | 22 +-
.../diskbalancer/command/PlanCommand.java | 63 +--
.../diskbalancer/command/QueryCommand.java | 19 +-
.../diskbalancer/command/ReportCommand.java | 18 +-
.../apache/hadoop/hdfs/tools/DiskBalancer.java | 482 -------------------
.../hadoop/hdfs/tools/DiskBalancerCLI.java | 482 +++++++++++++++++++
.../command/TestDiskBalancerCommand.java | 16 +-
11 files changed, 576 insertions(+), 572 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 5059528..7a90f08 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -127,7 +127,7 @@ function hdfscmd_case
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
diskbalancer)
- HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DiskBalancer
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DiskBalancerCLI
hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
index 8b83e27..007272e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
-import org.apache.hadoop.hdfs.tools.DiskBalancer;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import java.io.IOException;
@@ -44,9 +44,10 @@ public class CancelCommand extends Command {
*/
public CancelCommand(Configuration conf) {
super(conf);
- addValidCommandParameters(DiskBalancer.CANCEL, "Cancels a running plan.");
- addValidCommandParameters(DiskBalancer.NODE, "Node to run the command " +
- "against in node:port format.");
+ addValidCommandParameters(DiskBalancerCLI.CANCEL,
+ "Cancels a running plan.");
+ addValidCommandParameters(DiskBalancerCLI.NODE,
+ "Node to run the command against in node:port format.");
}
/**
@@ -57,20 +58,20 @@ public class CancelCommand extends Command {
@Override
public void execute(CommandLine cmd) throws Exception {
LOG.info("Executing \"Cancel plan\" command.");
- Preconditions.checkState(cmd.hasOption(DiskBalancer.CANCEL));
- verifyCommandOptions(DiskBalancer.CANCEL, cmd);
+ Preconditions.checkState(cmd.hasOption(DiskBalancerCLI.CANCEL));
+ verifyCommandOptions(DiskBalancerCLI.CANCEL, cmd);
// We can cancel a plan using datanode address and plan ID
// that you can read from a datanode using queryStatus
- if(cmd.hasOption(DiskBalancer.NODE)) {
- String nodeAddress = cmd.getOptionValue(DiskBalancer.NODE);
- String planHash = cmd.getOptionValue(DiskBalancer.CANCEL);
+ if(cmd.hasOption(DiskBalancerCLI.NODE)) {
+ String nodeAddress = cmd.getOptionValue(DiskBalancerCLI.NODE);
+ String planHash = cmd.getOptionValue(DiskBalancerCLI.CANCEL);
cancelPlanUsingHash(nodeAddress, planHash);
} else {
// Or you can cancel a plan using the plan file. If the user
// points us to the plan file, we can compute the hash as well as read
// the address of the datanode from the plan file.
- String planFile = cmd.getOptionValue(DiskBalancer.CANCEL);
+ String planFile = cmd.getOptionValue(DiskBalancerCLI.CANCEL);
Preconditions.checkArgument(planFile != null && !planFile.isEmpty(),
"Invalid plan file specified.");
String planData = null;
@@ -142,6 +143,6 @@ public class CancelCommand extends Command {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp("hdfs diskbalancer -cancel <planFile> | -cancel " +
"<planID> -node <hostname>",
- header, DiskBalancer.getCancelOptions(), footer);
+ header, DiskBalancerCLI.getCancelOptions(), footer);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index 5acd0ac..7641b44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet;
-import org.apache.hadoop.hdfs.tools.DiskBalancer;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.codehaus.jackson.map.ObjectMapper;
@@ -418,7 +418,7 @@ public abstract class Command extends Configured {
* @return default top number of nodes.
*/
protected int getDefaultTop() {
- return DiskBalancer.DEFAULT_TOP;
+ return DiskBalancerCLI.DEFAULT_TOP;
}
/**
@@ -437,7 +437,7 @@ public abstract class Command extends Configured {
protected int parseTopNodes(final CommandLine cmd, final StrBuilder result) {
String outputLine = "";
int nodes = 0;
- final String topVal = cmd.getOptionValue(DiskBalancer.TOP);
+ final String topVal = cmd.getOptionValue(DiskBalancerCLI.TOP);
if (StringUtils.isBlank(topVal)) {
outputLine = String.format(
"No top limit specified, using default top value %d.",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
index f363c34..3a348c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
-import org.apache.hadoop.hdfs.tools.DiskBalancer;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import java.io.IOException;
@@ -46,7 +46,8 @@ public class ExecuteCommand extends Command {
*/
public ExecuteCommand(Configuration conf) {
super(conf);
- addValidCommandParameters(DiskBalancer.EXECUTE, "Executes a given plan.");
+ addValidCommandParameters(DiskBalancerCLI.EXECUTE,
+ "Executes a given plan.");
}
/**
@@ -57,10 +58,10 @@ public class ExecuteCommand extends Command {
@Override
public void execute(CommandLine cmd) throws Exception {
LOG.info("Executing \"execute plan\" command");
- Preconditions.checkState(cmd.hasOption(DiskBalancer.EXECUTE));
- verifyCommandOptions(DiskBalancer.EXECUTE, cmd);
+ Preconditions.checkState(cmd.hasOption(DiskBalancerCLI.EXECUTE));
+ verifyCommandOptions(DiskBalancerCLI.EXECUTE, cmd);
- String planFile = cmd.getOptionValue(DiskBalancer.EXECUTE);
+ String planFile = cmd.getOptionValue(DiskBalancerCLI.EXECUTE);
Preconditions.checkArgument(planFile != null && !planFile.isEmpty(),
"Invalid plan file specified.");
@@ -88,7 +89,7 @@ public class ExecuteCommand extends Command {
String planHash = DigestUtils.shaHex(planData);
try {
// TODO : Support skipping date check.
- dataNode.submitDiskBalancerPlan(planHash, DiskBalancer.PLAN_VERSION,
+ dataNode.submitDiskBalancerPlan(planHash, DiskBalancerCLI.PLAN_VERSION,
planFile, planData, false);
} catch (DiskBalancerException ex) {
LOG.error("Submitting plan on {} failed. Result: {}, Message: {}",
@@ -111,6 +112,6 @@ public class ExecuteCommand extends Command {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp("hdfs diskbalancer -execute <planfile>",
- header, DiskBalancer.getExecuteOptions(), footer);
+ header, DiskBalancerCLI.getExecuteOptions(), footer);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java
index 3c2fd0c..c735299 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java
@@ -23,7 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.tools.DiskBalancer;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
/**
* Help Command prints out detailed help about each command.
@@ -37,7 +37,7 @@ public class HelpCommand extends Command {
*/
public HelpCommand(Configuration conf) {
super(conf);
- addValidCommandParameters(DiskBalancer.HELP, "Help Command");
+ addValidCommandParameters(DiskBalancerCLI.HELP, "Help Command");
}
/**
@@ -53,9 +53,9 @@ public class HelpCommand extends Command {
return;
}
- Preconditions.checkState(cmd.hasOption(DiskBalancer.HELP));
- verifyCommandOptions(DiskBalancer.HELP, cmd);
- String helpCommand = cmd.getOptionValue(DiskBalancer.HELP);
+ Preconditions.checkState(cmd.hasOption(DiskBalancerCLI.HELP));
+ verifyCommandOptions(DiskBalancerCLI.HELP, cmd);
+ String helpCommand = cmd.getOptionValue(DiskBalancerCLI.HELP);
if (helpCommand == null || helpCommand.isEmpty()) {
this.printHelp();
return;
@@ -65,19 +65,19 @@ public class HelpCommand extends Command {
helpCommand = helpCommand.toLowerCase();
Command command = null;
switch (helpCommand) {
- case DiskBalancer.PLAN:
+ case DiskBalancerCLI.PLAN:
command = new PlanCommand(getConf());
break;
- case DiskBalancer.EXECUTE:
+ case DiskBalancerCLI.EXECUTE:
command = new ExecuteCommand(getConf());
break;
- case DiskBalancer.QUERY:
+ case DiskBalancerCLI.QUERY:
command = new QueryCommand(getConf());
break;
- case DiskBalancer.CANCEL:
+ case DiskBalancerCLI.CANCEL:
command = new CancelCommand(getConf());
break;
- case DiskBalancer.REPORT:
+ case DiskBalancerCLI.REPORT:
command = new ReportCommand(getConf(), null);
break;
default:
@@ -102,7 +102,7 @@ public class HelpCommand extends Command {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp("hdfs diskbalancer [command] [options]",
- header, DiskBalancer.getHelpOptions(), "");
+ header, DiskBalancerCLI.getHelpOptions(), "");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
index 72ad2c6..9749409 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hdfs.server.diskbalancer.datamodel
.DiskBalancerDataNode;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.Step;
-import org.apache.hadoop.hdfs.tools.DiskBalancer;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -53,18 +53,18 @@ public class PlanCommand extends Command {
this.thresholdPercentage = 1;
this.bandwidth = 0;
this.maxError = 0;
- addValidCommandParameters(DiskBalancer.OUTFILE, "Output directory in " +
+ addValidCommandParameters(DiskBalancerCLI.OUTFILE, "Output directory in " +
"HDFS. The generated plan will be written to a file in this " +
"directory.");
- addValidCommandParameters(DiskBalancer.BANDWIDTH, "Maximum Bandwidth to " +
- "be used while copying.");
- addValidCommandParameters(DiskBalancer.THRESHOLD, "Percentage skew that " +
- "we tolerate before diskbalancer starts working.");
- addValidCommandParameters(DiskBalancer.MAXERROR, "Max errors to tolerate " +
- "between 2 disks");
- addValidCommandParameters(DiskBalancer.VERBOSE, "Run plan command in " +
+ addValidCommandParameters(DiskBalancerCLI.BANDWIDTH,
+ "Maximum Bandwidth to be used while copying.");
+ addValidCommandParameters(DiskBalancerCLI.THRESHOLD,
+ "Percentage skew that we tolerate before diskbalancer starts working.");
+ addValidCommandParameters(DiskBalancerCLI.MAXERROR,
+ "Max errors to tolerate between 2 disks");
+ addValidCommandParameters(DiskBalancerCLI.VERBOSE, "Run plan command in " +
"verbose mode.");
- addValidCommandParameters(DiskBalancer.PLAN, "Plan Command");
+ addValidCommandParameters(DiskBalancerCLI.PLAN, "Plan Command");
}
/**
@@ -77,36 +77,37 @@ public class PlanCommand extends Command {
@Override
public void execute(CommandLine cmd) throws Exception {
LOG.debug("Processing Plan Command.");
- Preconditions.checkState(cmd.hasOption(DiskBalancer.PLAN));
- verifyCommandOptions(DiskBalancer.PLAN, cmd);
+ Preconditions.checkState(cmd.hasOption(DiskBalancerCLI.PLAN));
+ verifyCommandOptions(DiskBalancerCLI.PLAN, cmd);
- if (cmd.getOptionValue(DiskBalancer.PLAN) == null) {
+ if (cmd.getOptionValue(DiskBalancerCLI.PLAN) == null) {
throw new IllegalArgumentException("A node name is required to create a" +
" plan.");
}
- if (cmd.hasOption(DiskBalancer.BANDWIDTH)) {
- this.bandwidth = Integer.parseInt(cmd.getOptionValue(DiskBalancer
+ if (cmd.hasOption(DiskBalancerCLI.BANDWIDTH)) {
+ this.bandwidth = Integer.parseInt(cmd.getOptionValue(DiskBalancerCLI
.BANDWIDTH));
}
- if (cmd.hasOption(DiskBalancer.MAXERROR)) {
- this.maxError = Integer.parseInt(cmd.getOptionValue(DiskBalancer
+ if (cmd.hasOption(DiskBalancerCLI.MAXERROR)) {
+ this.maxError = Integer.parseInt(cmd.getOptionValue(DiskBalancerCLI
.MAXERROR));
}
readClusterInfo(cmd);
String output = null;
- if (cmd.hasOption(DiskBalancer.OUTFILE)) {
- output = cmd.getOptionValue(DiskBalancer.OUTFILE);
+ if (cmd.hasOption(DiskBalancerCLI.OUTFILE)) {
+ output = cmd.getOptionValue(DiskBalancerCLI.OUTFILE);
}
setOutputPath(output);
// -plan nodename is the command line argument.
- DiskBalancerDataNode node = getNode(cmd.getOptionValue(DiskBalancer.PLAN));
+ DiskBalancerDataNode node =
+ getNode(cmd.getOptionValue(DiskBalancerCLI.PLAN));
if (node == null) {
throw new IllegalArgumentException("Unable to find the specified node. " +
- cmd.getOptionValue(DiskBalancer.PLAN));
+ cmd.getOptionValue(DiskBalancerCLI.PLAN));
}
this.thresholdPercentage = getThresholdPercentage(cmd);
@@ -124,8 +125,8 @@ public class PlanCommand extends Command {
try (FSDataOutputStream beforeStream = create(String.format(
- DiskBalancer.BEFORE_TEMPLATE,
- cmd.getOptionValue(DiskBalancer.PLAN)))) {
+ DiskBalancerCLI.BEFORE_TEMPLATE,
+ cmd.getOptionValue(DiskBalancerCLI.PLAN)))) {
beforeStream.write(getCluster().toJson()
.getBytes(StandardCharsets.UTF_8));
}
@@ -133,17 +134,17 @@ public class PlanCommand extends Command {
if (plan != null && plan.getVolumeSetPlans().size() > 0) {
LOG.info("Writing plan to : {}", getOutputPath());
try (FSDataOutputStream planStream = create(String.format(
- DiskBalancer.PLAN_TEMPLATE,
- cmd.getOptionValue(DiskBalancer.PLAN)))) {
+ DiskBalancerCLI.PLAN_TEMPLATE,
+ cmd.getOptionValue(DiskBalancerCLI.PLAN)))) {
planStream.write(plan.toJson().getBytes(StandardCharsets.UTF_8));
}
} else {
LOG.info("No plan generated. DiskBalancing not needed for node: {} " +
- "threshold used: {}", cmd.getOptionValue(DiskBalancer.PLAN),
+ "threshold used: {}", cmd.getOptionValue(DiskBalancerCLI.PLAN),
this.thresholdPercentage);
}
- if (cmd.hasOption(DiskBalancer.VERBOSE) && plans.size() > 0) {
+ if (cmd.hasOption(DiskBalancerCLI.VERBOSE) && plans.size() > 0) {
printToScreen(plans);
}
}
@@ -162,8 +163,8 @@ public class PlanCommand extends Command {
" will balance the data.";
HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp("hdfs diskbalancer -plan " +
- "<hostname> [options]", header, DiskBalancer.getPlanOptions(), footer);
+ helpFormatter.printHelp("hdfs diskbalancer -plan <hostname> [options]",
+ header, DiskBalancerCLI.getPlanOptions(), footer);
}
/**
@@ -174,8 +175,8 @@ public class PlanCommand extends Command {
*/
private double getThresholdPercentage(CommandLine cmd) {
Double value = 0.0;
- if (cmd.hasOption(DiskBalancer.THRESHOLD)) {
- value = Double.parseDouble(cmd.getOptionValue(DiskBalancer.THRESHOLD));
+ if (cmd.hasOption(DiskBalancerCLI.THRESHOLD)) {
+ value = Double.parseDouble(cmd.getOptionValue(DiskBalancerCLI.THRESHOLD));
}
if ((value <= 0.0) || (value > 100.0)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java
index 1557a02..a8adcbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus;
import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
-import org.apache.hadoop.hdfs.tools.DiskBalancer;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import org.apache.hadoop.net.NetUtils;
/**
@@ -42,9 +42,10 @@ public class QueryCommand extends Command {
*/
public QueryCommand(Configuration conf) {
super(conf);
- addValidCommandParameters(DiskBalancer.QUERY, "Queries the status of disk" +
- " plan running on a given datanode.");
- addValidCommandParameters(DiskBalancer.VERBOSE, "Prints verbose results.");
+ addValidCommandParameters(DiskBalancerCLI.QUERY,
+ "Queries the status of disk plan running on a given datanode.");
+ addValidCommandParameters(DiskBalancerCLI.VERBOSE,
+ "Prints verbose results.");
}
/**
@@ -55,9 +56,9 @@ public class QueryCommand extends Command {
@Override
public void execute(CommandLine cmd) throws Exception {
LOG.info("Executing \"query plan\" command.");
- Preconditions.checkState(cmd.hasOption(DiskBalancer.QUERY));
- verifyCommandOptions(DiskBalancer.QUERY, cmd);
- String nodeName = cmd.getOptionValue(DiskBalancer.QUERY);
+ Preconditions.checkState(cmd.hasOption(DiskBalancerCLI.QUERY));
+ verifyCommandOptions(DiskBalancerCLI.QUERY, cmd);
+ String nodeName = cmd.getOptionValue(DiskBalancerCLI.QUERY);
Preconditions.checkNotNull(nodeName);
nodeName = nodeName.trim();
String nodeAddress = nodeName;
@@ -79,7 +80,7 @@ public class QueryCommand extends Command {
workStatus.getPlanID(),
workStatus.getResult().toString());
- if (cmd.hasOption(DiskBalancer.VERBOSE)) {
+ if (cmd.hasOption(DiskBalancerCLI.VERBOSE)) {
System.out.printf("%s", workStatus.currentStateString());
}
} catch (DiskBalancerException ex) {
@@ -101,6 +102,6 @@ public class QueryCommand extends Command {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp("hdfs diskbalancer -query <hostname> [options]",
- header, DiskBalancer.getQueryOptions(), footer);
+ header, DiskBalancerCLI.getQueryOptions(), footer);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
index 18dd77e..69b765f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet;
-import org.apache.hadoop.hdfs.tools.DiskBalancer;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -52,15 +52,15 @@ public class ReportCommand extends Command {
super(conf);
this.out = out;
- addValidCommandParameters(DiskBalancer.REPORT,
+ addValidCommandParameters(DiskBalancerCLI.REPORT,
"Report volume information of nodes.");
String desc = String.format(
"Top number of nodes to be processed. Default: %d", getDefaultTop());
- addValidCommandParameters(DiskBalancer.TOP, desc);
+ addValidCommandParameters(DiskBalancerCLI.TOP, desc);
desc = String.format("Print out volume information for a DataNode.");
- addValidCommandParameters(DiskBalancer.NODE, desc);
+ addValidCommandParameters(DiskBalancerCLI.NODE, desc);
}
@Override
@@ -69,8 +69,8 @@ public class ReportCommand extends Command {
String outputLine = "Processing report command";
recordOutput(result, outputLine);
- Preconditions.checkState(cmd.hasOption(DiskBalancer.REPORT));
- verifyCommandOptions(DiskBalancer.REPORT, cmd);
+ Preconditions.checkState(cmd.hasOption(DiskBalancerCLI.REPORT));
+ verifyCommandOptions(DiskBalancerCLI.REPORT, cmd);
readClusterInfo(cmd);
final String nodeFormat =
@@ -81,7 +81,7 @@ public class ReportCommand extends Command {
"[%s: volume-%s] - %.2f used: %d/%d, %.2f free: %d/%d, "
+ "isFailed: %s, isReadOnly: %s, isSkip: %s, isTransient: %s.";
- if (cmd.hasOption(DiskBalancer.NODE)) {
+ if (cmd.hasOption(DiskBalancerCLI.NODE)) {
/*
* Reporting volume information for a specific DataNode
*/
@@ -136,7 +136,7 @@ public class ReportCommand extends Command {
* get value that identifies a DataNode from command line, it could be UUID,
* IP address or host name.
*/
- final String nodeVal = cmd.getOptionValue(DiskBalancer.NODE);
+ final String nodeVal = cmd.getOptionValue(DiskBalancerCLI.NODE);
if (StringUtils.isBlank(nodeVal)) {
outputLine = "The value for '-node' is neither specified or empty.";
@@ -211,6 +211,6 @@ public class ReportCommand extends Command {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp("hdfs diskbalancer -fs http://namenode.uri " +
"-report [options]",
- header, DiskBalancer.getReportOptions(), footer);
+ header, DiskBalancerCLI.getReportOptions(), footer);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
deleted file mode 100644
index 1ed2fdc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
+++ /dev/null
@@ -1,482 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdfs.tools;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.server.diskbalancer.command.CancelCommand;
-import org.apache.hadoop.hdfs.server.diskbalancer.command.Command;
-import org.apache.hadoop.hdfs.server.diskbalancer.command.ExecuteCommand;
-import org.apache.hadoop.hdfs.server.diskbalancer.command.HelpCommand;
-import org.apache.hadoop.hdfs.server.diskbalancer.command.PlanCommand;
-import org.apache.hadoop.hdfs.server.diskbalancer.command.QueryCommand;
-import org.apache.hadoop.hdfs.server.diskbalancer.command.ReportCommand;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.PrintStream;
-
-/**
- * DiskBalancer is a tool that can be used to ensure that data is spread evenly
- * across volumes of same storage type.
- * <p>
- * For example, if you have 3 disks, with 100 GB , 600 GB and 200 GB on each
- * disk, this tool will ensure that each disk will have 300 GB.
- * <p>
- * This tool can be run while data nodes are fully functional.
- * <p>
- * At very high level diskbalancer computes a set of moves that will make disk
- * utilization equal and then those moves are executed by the datanode.
- */
-public class DiskBalancer extends Configured implements Tool {
- /**
- * Computes a plan for a given set of nodes.
- */
- public static final String PLAN = "plan";
- /**
- * Output file name, for commands like report, plan etc. This is an optional
- * argument, by default diskbalancer will write all its output to
- * /system/reports/diskbalancer of the current cluster it is operating
- * against.
- */
- public static final String OUTFILE = "out";
- /**
- * Help for the program.
- */
- public static final String HELP = "help";
- /**
- * Percentage of data unevenness that we are willing to live with. For example
- * - a value like 10 indicates that we are okay with 10 % +/- from
- * idealStorage Target.
- */
- public static final String THRESHOLD = "thresholdPercentage";
- /**
- * Specifies the maximum disk bandwidth to use per second.
- */
- public static final String BANDWIDTH = "bandwidth";
- /**
- * Specifies the maximum errors to tolerate.
- */
- public static final String MAXERROR = "maxerror";
- /**
- * Executes a given plan file on the target datanode.
- */
- public static final String EXECUTE = "execute";
- /**
- * The report command prints out a disk fragmentation report about the data
- * cluster. By default it prints the DEFAULT_TOP machines names with high
- * nodeDataDensity {DiskBalancerDataNode#getNodeDataDensity} values. This
- * means that these are the nodes that deviates from the ideal data
- * distribution.
- */
- public static final String REPORT = "report";
- /**
- * specify top number of nodes to be processed.
- */
- public static final String TOP = "top";
- /**
- * specify default top number of nodes to be processed.
- */
- public static final int DEFAULT_TOP = 100;
- /**
- * Name or address of the node to execute against.
- */
- public static final String NODE = "node";
- /**
- * Runs the command in verbose mode.
- */
- public static final String VERBOSE = "v";
- public static final int PLAN_VERSION = 1;
- /**
- * Reports the status of disk balancer operation.
- */
- public static final String QUERY = "query";
- /**
- * Cancels a running plan.
- */
- public static final String CANCEL = "cancel";
- /**
- * Template for the Before File. It is node.before.json.
- */
- public static final String BEFORE_TEMPLATE = "%s.before.json";
- /**
- * Template for the plan file. it is node.plan.json.
- */
- public static final String PLAN_TEMPLATE = "%s.plan.json";
- private static final Logger LOG =
- LoggerFactory.getLogger(DiskBalancer.class);
-
- private static final Options PLAN_OPTIONS = new Options();
- private static final Options EXECUTE_OPTIONS = new Options();
- private static final Options QUERY_OPTIONS = new Options();
- private static final Options HELP_OPTIONS = new Options();
- private static final Options CANCEL_OPTIONS = new Options();
- private static final Options REPORT_OPTIONS = new Options();
-
- /**
- * Construct a DiskBalancer.
- *
- * @param conf
- */
- public DiskBalancer(Configuration conf) {
- super(conf);
- }
-
- /**
- * Main for the DiskBalancer Command handling.
- *
- * @param argv - System Args Strings[]
- * @throws Exception
- */
- public static void main(String[] argv) throws Exception {
- DiskBalancer shell = new DiskBalancer(new HdfsConfiguration());
- int res = 0;
- try {
- res = ToolRunner.run(shell, argv);
- } catch (Exception ex) {
- LOG.error(ex.toString());
- res = 1;
- }
- System.exit(res);
- }
-
- /**
- * Execute the command with the given arguments.
- *
- * @param args command specific arguments.
- * @return exit code.
- * @throws Exception
- */
- @Override
- public int run(String[] args) throws Exception {
- return run(args, System.out);
- }
-
- /**
- * Execute the command with the given arguments.
- *
- * @param args command specific arguments.
- * @param out the output stream used for printing
- * @return exit code.
- * @throws Exception
- */
- public int run(String[] args, final PrintStream out) throws Exception {
- Options opts = getOpts();
- CommandLine cmd = parseArgs(args, opts);
- return dispatch(cmd, opts, out);
- }
-
- /**
- * returns the Command Line Options.
- *
- * @return Options
- */
- private Options getOpts() {
- Options opts = new Options();
- addPlanCommands(opts);
- addHelpCommands(opts);
- addExecuteCommands(opts);
- addQueryCommands(opts);
- addCancelCommands(opts);
- addReportCommands(opts);
- return opts;
- }
-
- /**
- * Returns Plan options.
- *
- * @return Options.
- */
- public static Options getPlanOptions() {
- return PLAN_OPTIONS;
- }
-
- /**
- * Returns help options.
- *
- * @return - help options.
- */
- public static Options getHelpOptions() {
- return HELP_OPTIONS;
- }
-
- /**
- * Retuns execute options.
- *
- * @return - execute options.
- */
- public static Options getExecuteOptions() {
- return EXECUTE_OPTIONS;
- }
-
- /**
- * Returns Query Options.
- *
- * @return query Options
- */
- public static Options getQueryOptions() {
- return QUERY_OPTIONS;
- }
-
- /**
- * Returns Cancel Options.
- *
- * @return Options
- */
- public static Options getCancelOptions() {
- return CANCEL_OPTIONS;
- }
-
- /**
- * Returns Report Options.
- *
- * @return Options
- */
- public static Options getReportOptions() {
- return REPORT_OPTIONS;
- }
-
- /**
- * Adds commands for plan command.
- *
- * @return Options.
- */
- private void addPlanCommands(Options opt) {
-
- Option plan = OptionBuilder.withLongOpt(PLAN)
- .withDescription("Hostname, IP address or UUID of datanode " +
- "for which a plan is created.")
- .hasArg()
- .create();
- getPlanOptions().addOption(plan);
- opt.addOption(plan);
-
-
- Option outFile = OptionBuilder.withLongOpt(OUTFILE).hasArg()
- .withDescription(
- "Local path of file to write output to, if not specified "
- + "defaults will be used.")
- .create();
- getPlanOptions().addOption(outFile);
- opt.addOption(outFile);
-
- Option bandwidth = OptionBuilder.withLongOpt(BANDWIDTH).hasArg()
- .withDescription(
- "Maximum disk bandwidth (MB/s) in integer to be consumed by "
- + "diskBalancer. e.g. 10 MB/s.")
- .create();
- getPlanOptions().addOption(bandwidth);
- opt.addOption(bandwidth);
-
- Option threshold = OptionBuilder.withLongOpt(THRESHOLD)
- .hasArg()
- .withDescription("Percentage of data skew that is tolerated before"
- + " disk balancer starts working. For example, if"
- + " total data on a 2 disk node is 100 GB then disk"
- + " balancer calculates the expected value on each disk,"
- + " which is 50 GB. If the tolerance is 10% then data"
- + " on a single disk needs to be more than 60 GB"
- + " (50 GB + 10% tolerance value) for Disk balancer to"
- + " balance the disks.")
- .create();
- getPlanOptions().addOption(threshold);
- opt.addOption(threshold);
-
-
- Option maxError = OptionBuilder.withLongOpt(MAXERROR)
- .hasArg()
- .withDescription("Describes how many errors " +
- "can be tolerated while copying between a pair of disks.")
- .create();
- getPlanOptions().addOption(maxError);
- opt.addOption(maxError);
-
- Option verbose = OptionBuilder.withLongOpt(VERBOSE)
- .withDescription("Print out the summary of the plan on console")
- .create();
- getPlanOptions().addOption(verbose);
- opt.addOption(verbose);
- }
-
- /**
- * Adds Help to the options.
- */
- private void addHelpCommands(Options opt) {
- Option help = OptionBuilder.withLongOpt(HELP)
- .hasOptionalArg()
- .withDescription("valid commands are plan | execute | query | cancel" +
- " | report")
- .create();
- getHelpOptions().addOption(help);
- opt.addOption(help);
- }
-
- /**
- * Adds execute command options.
- *
- * @param opt Options
- */
- private void addExecuteCommands(Options opt) {
- Option execute = OptionBuilder.withLongOpt(EXECUTE)
- .hasArg()
- .withDescription("Takes a plan file and " +
- "submits it for execution by the datanode.")
- .create();
- getExecuteOptions().addOption(execute);
- opt.addOption(execute);
- }
-
- /**
- * Adds query command options.
- *
- * @param opt Options
- */
- private void addQueryCommands(Options opt) {
- Option query = OptionBuilder.withLongOpt(QUERY)
- .hasArg()
- .withDescription("Queries the disk balancer " +
- "status of a given datanode.")
- .create();
- getQueryOptions().addOption(query);
- opt.addOption(query);
-
- // Please note: Adding this only to Query options since -v is already
- // added to global table.
- Option verbose = OptionBuilder.withLongOpt(VERBOSE)
- .withDescription("Prints details of the plan that is being executed " +
- "on the node.")
- .create();
- getQueryOptions().addOption(verbose);
- }
-
- /**
- * Adds cancel command options.
- *
- * @param opt Options
- */
- private void addCancelCommands(Options opt) {
- Option cancel = OptionBuilder.withLongOpt(CANCEL)
- .hasArg()
- .withDescription("Cancels a running plan using a plan file.")
- .create();
- getCancelOptions().addOption(cancel);
- opt.addOption(cancel);
-
- Option node = OptionBuilder.withLongOpt(NODE)
- .hasArg()
- .withDescription("Cancels a running plan using a plan ID and hostName")
- .create();
-
- getCancelOptions().addOption(node);
- opt.addOption(node);
- }
-
- /**
- * Adds report command options.
- *
- * @param opt Options
- */
- private void addReportCommands(Options opt) {
- Option report = OptionBuilder.withLongOpt(REPORT)
- .withDescription("List nodes that will benefit from running " +
- "DiskBalancer.")
- .create();
- getReportOptions().addOption(report);
- opt.addOption(report);
-
- Option top = OptionBuilder.withLongOpt(TOP)
- .hasArg()
- .withDescription("specify the number of nodes to be listed which has" +
- " data imbalance.")
- .create();
- getReportOptions().addOption(top);
- opt.addOption(top);
-
- Option node = OptionBuilder.withLongOpt(NODE)
- .hasArg()
- .withDescription("Datanode address, " +
- "it can be DataNodeID, IP or hostname.")
- .create();
- getReportOptions().addOption(node);
- opt.addOption(node);
- }
-
- /**
- * This function parses all command line arguments and returns the appropriate
- * values.
- *
- * @param argv - Argv from main
- * @return CommandLine
- */
- private CommandLine parseArgs(String[] argv, Options opts)
- throws org.apache.commons.cli.ParseException {
- BasicParser parser = new BasicParser();
- return parser.parse(opts, argv);
- }
-
- /**
- * Dispatches calls to the right command Handler classes.
- *
- * @param cmd - CommandLine
- * @param opts options of command line
- * @param out the output stream used for printing
- */
- private int dispatch(CommandLine cmd, Options opts, final PrintStream out)
- throws Exception {
- Command currentCommand = null;
- if (cmd.hasOption(DiskBalancer.PLAN)) {
- currentCommand = new PlanCommand(getConf());
- }
-
- if (cmd.hasOption(DiskBalancer.EXECUTE)) {
- currentCommand = new ExecuteCommand(getConf());
- }
-
- if (cmd.hasOption(DiskBalancer.QUERY)) {
- currentCommand = new QueryCommand(getConf());
- }
-
- if (cmd.hasOption(DiskBalancer.CANCEL)) {
- currentCommand = new CancelCommand(getConf());
- }
-
- if (cmd.hasOption(DiskBalancer.REPORT)) {
- currentCommand = new ReportCommand(getConf(), out);
- }
-
- if (cmd.hasOption(DiskBalancer.HELP)) {
- currentCommand = new HelpCommand(getConf());
- }
-
- // Invoke main help here.
- if (currentCommand == null) {
- new HelpCommand(getConf()).execute(null);
- return 1;
- }
-
- currentCommand.execute(cmd);
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancerCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancerCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancerCLI.java
new file mode 100644
index 0000000..e961c14
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancerCLI.java
@@ -0,0 +1,482 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.CancelCommand;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.Command;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.ExecuteCommand;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.HelpCommand;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.PlanCommand;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.QueryCommand;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.ReportCommand;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.PrintStream;
+
+/**
+ * DiskBalancer is a tool that can be used to ensure that data is spread evenly
+ * across volumes of same storage type.
+ * <p>
+ * For example, if you have 3 disks, with 100 GB , 600 GB and 200 GB on each
+ * disk, this tool will ensure that each disk will have 300 GB.
+ * <p>
+ * This tool can be run while data nodes are fully functional.
+ * <p>
+ * At very high level diskbalancer computes a set of moves that will make disk
+ * utilization equal and then those moves are executed by the datanode.
+ */
+public class DiskBalancerCLI extends Configured implements Tool {
+ /**
+ * Computes a plan for a given set of nodes.
+ */
+ public static final String PLAN = "plan";
+ /**
+ * Output file name, for commands like report, plan etc. This is an optional
+ * argument, by default diskbalancer will write all its output to
+ * /system/reports/diskbalancer of the current cluster it is operating
+ * against.
+ */
+ public static final String OUTFILE = "out";
+ /**
+ * Help for the program.
+ */
+ public static final String HELP = "help";
+ /**
+ * Percentage of data unevenness that we are willing to live with. For example
+ * - a value like 10 indicates that we are okay with 10 % +/- from
+ * idealStorage Target.
+ */
+ public static final String THRESHOLD = "thresholdPercentage";
+ /**
+ * Specifies the maximum disk bandwidth to use per second.
+ */
+ public static final String BANDWIDTH = "bandwidth";
+ /**
+ * Specifies the maximum errors to tolerate.
+ */
+ public static final String MAXERROR = "maxerror";
+ /**
+ * Executes a given plan file on the target datanode.
+ */
+ public static final String EXECUTE = "execute";
+ /**
+ * The report command prints out a disk fragmentation report about the data
+ * cluster. By default it prints the DEFAULT_TOP machines names with high
+ * nodeDataDensity {DiskBalancerDataNode#getNodeDataDensity} values. This
+ * means that these are the nodes that deviates from the ideal data
+ * distribution.
+ */
+ public static final String REPORT = "report";
+ /**
+ * specify top number of nodes to be processed.
+ */
+ public static final String TOP = "top";
+ /**
+ * specify default top number of nodes to be processed.
+ */
+ public static final int DEFAULT_TOP = 100;
+ /**
+ * Name or address of the node to execute against.
+ */
+ public static final String NODE = "node";
+ /**
+ * Runs the command in verbose mode.
+ */
+ public static final String VERBOSE = "v";
+ public static final int PLAN_VERSION = 1;
+ /**
+ * Reports the status of disk balancer operation.
+ */
+ public static final String QUERY = "query";
+ /**
+ * Cancels a running plan.
+ */
+ public static final String CANCEL = "cancel";
+ /**
+ * Template for the Before File. It is node.before.json.
+ */
+ public static final String BEFORE_TEMPLATE = "%s.before.json";
+ /**
+ * Template for the plan file. it is node.plan.json.
+ */
+ public static final String PLAN_TEMPLATE = "%s.plan.json";
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DiskBalancerCLI.class);
+
+ private static final Options PLAN_OPTIONS = new Options();
+ private static final Options EXECUTE_OPTIONS = new Options();
+ private static final Options QUERY_OPTIONS = new Options();
+ private static final Options HELP_OPTIONS = new Options();
+ private static final Options CANCEL_OPTIONS = new Options();
+ private static final Options REPORT_OPTIONS = new Options();
+
+ /**
+ * Construct a DiskBalancer.
+ *
+ * @param conf
+ */
+ public DiskBalancerCLI(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * Main for the DiskBalancer Command handling.
+ *
+ * @param argv - System Args Strings[]
+ * @throws Exception
+ */
+ public static void main(String[] argv) throws Exception {
+ DiskBalancerCLI shell = new DiskBalancerCLI(new HdfsConfiguration());
+ int res = 0;
+ try {
+ res = ToolRunner.run(shell, argv);
+ } catch (Exception ex) {
+ LOG.error(ex.toString());
+ res = 1;
+ }
+ System.exit(res);
+ }
+
+ /**
+ * Execute the command with the given arguments.
+ *
+ * @param args command specific arguments.
+ * @return exit code.
+ * @throws Exception
+ */
+ @Override
+ public int run(String[] args) throws Exception {
+ return run(args, System.out);
+ }
+
+ /**
+ * Execute the command with the given arguments.
+ *
+ * @param args command specific arguments.
+ * @param out the output stream used for printing
+ * @return exit code.
+ * @throws Exception
+ */
+ public int run(String[] args, final PrintStream out) throws Exception {
+ Options opts = getOpts();
+ CommandLine cmd = parseArgs(args, opts);
+ return dispatch(cmd, opts, out);
+ }
+
+ /**
+ * returns the Command Line Options.
+ *
+ * @return Options
+ */
+ private Options getOpts() {
+ Options opts = new Options();
+ addPlanCommands(opts);
+ addHelpCommands(opts);
+ addExecuteCommands(opts);
+ addQueryCommands(opts);
+ addCancelCommands(opts);
+ addReportCommands(opts);
+ return opts;
+ }
+
+ /**
+ * Returns Plan options.
+ *
+ * @return Options.
+ */
+ public static Options getPlanOptions() {
+ return PLAN_OPTIONS;
+ }
+
+ /**
+ * Returns help options.
+ *
+ * @return - help options.
+ */
+ public static Options getHelpOptions() {
+ return HELP_OPTIONS;
+ }
+
+ /**
+ * Retuns execute options.
+ *
+ * @return - execute options.
+ */
+ public static Options getExecuteOptions() {
+ return EXECUTE_OPTIONS;
+ }
+
+ /**
+ * Returns Query Options.
+ *
+ * @return query Options
+ */
+ public static Options getQueryOptions() {
+ return QUERY_OPTIONS;
+ }
+
+ /**
+ * Returns Cancel Options.
+ *
+ * @return Options
+ */
+ public static Options getCancelOptions() {
+ return CANCEL_OPTIONS;
+ }
+
+ /**
+ * Returns Report Options.
+ *
+ * @return Options
+ */
+ public static Options getReportOptions() {
+ return REPORT_OPTIONS;
+ }
+
+ /**
+ * Adds commands for plan command.
+ *
+ * @return Options.
+ */
+ private void addPlanCommands(Options opt) {
+
+ Option plan = OptionBuilder.withLongOpt(PLAN)
+ .withDescription("Hostname, IP address or UUID of datanode " +
+ "for which a plan is created.")
+ .hasArg()
+ .create();
+ getPlanOptions().addOption(plan);
+ opt.addOption(plan);
+
+
+ Option outFile = OptionBuilder.withLongOpt(OUTFILE).hasArg()
+ .withDescription(
+ "Local path of file to write output to, if not specified "
+ + "defaults will be used.")
+ .create();
+ getPlanOptions().addOption(outFile);
+ opt.addOption(outFile);
+
+ Option bandwidth = OptionBuilder.withLongOpt(BANDWIDTH).hasArg()
+ .withDescription(
+ "Maximum disk bandwidth (MB/s) in integer to be consumed by "
+ + "diskBalancer. e.g. 10 MB/s.")
+ .create();
+ getPlanOptions().addOption(bandwidth);
+ opt.addOption(bandwidth);
+
+ Option threshold = OptionBuilder.withLongOpt(THRESHOLD)
+ .hasArg()
+ .withDescription("Percentage of data skew that is tolerated before"
+ + " disk balancer starts working. For example, if"
+ + " total data on a 2 disk node is 100 GB then disk"
+ + " balancer calculates the expected value on each disk,"
+ + " which is 50 GB. If the tolerance is 10% then data"
+ + " on a single disk needs to be more than 60 GB"
+ + " (50 GB + 10% tolerance value) for Disk balancer to"
+ + " balance the disks.")
+ .create();
+ getPlanOptions().addOption(threshold);
+ opt.addOption(threshold);
+
+
+ Option maxError = OptionBuilder.withLongOpt(MAXERROR)
+ .hasArg()
+ .withDescription("Describes how many errors " +
+ "can be tolerated while copying between a pair of disks.")
+ .create();
+ getPlanOptions().addOption(maxError);
+ opt.addOption(maxError);
+
+ Option verbose = OptionBuilder.withLongOpt(VERBOSE)
+ .withDescription("Print out the summary of the plan on console")
+ .create();
+ getPlanOptions().addOption(verbose);
+ opt.addOption(verbose);
+ }
+
+ /**
+ * Adds Help to the options.
+ */
+ private void addHelpCommands(Options opt) {
+ Option help = OptionBuilder.withLongOpt(HELP)
+ .hasOptionalArg()
+ .withDescription("valid commands are plan | execute | query | cancel" +
+ " | report")
+ .create();
+ getHelpOptions().addOption(help);
+ opt.addOption(help);
+ }
+
+ /**
+ * Adds execute command options.
+ *
+ * @param opt Options
+ */
+ private void addExecuteCommands(Options opt) {
+ Option execute = OptionBuilder.withLongOpt(EXECUTE)
+ .hasArg()
+ .withDescription("Takes a plan file and " +
+ "submits it for execution by the datanode.")
+ .create();
+ getExecuteOptions().addOption(execute);
+ opt.addOption(execute);
+ }
+
+ /**
+ * Adds query command options.
+ *
+ * @param opt Options
+ */
+ private void addQueryCommands(Options opt) {
+ Option query = OptionBuilder.withLongOpt(QUERY)
+ .hasArg()
+ .withDescription("Queries the disk balancer " +
+ "status of a given datanode.")
+ .create();
+ getQueryOptions().addOption(query);
+ opt.addOption(query);
+
+ // Please note: Adding this only to Query options since -v is already
+ // added to global table.
+ Option verbose = OptionBuilder.withLongOpt(VERBOSE)
+ .withDescription("Prints details of the plan that is being executed " +
+ "on the node.")
+ .create();
+ getQueryOptions().addOption(verbose);
+ }
+
+ /**
+ * Adds cancel command options.
+ *
+ * @param opt Options
+ */
+ private void addCancelCommands(Options opt) {
+ Option cancel = OptionBuilder.withLongOpt(CANCEL)
+ .hasArg()
+ .withDescription("Cancels a running plan using a plan file.")
+ .create();
+ getCancelOptions().addOption(cancel);
+ opt.addOption(cancel);
+
+ Option node = OptionBuilder.withLongOpt(NODE)
+ .hasArg()
+ .withDescription("Cancels a running plan using a plan ID and hostName")
+ .create();
+
+ getCancelOptions().addOption(node);
+ opt.addOption(node);
+ }
+
+ /**
+ * Adds report command options.
+ *
+ * @param opt Options
+ */
+ private void addReportCommands(Options opt) {
+ Option report = OptionBuilder.withLongOpt(REPORT)
+ .withDescription("List nodes that will benefit from running " +
+ "DiskBalancer.")
+ .create();
+ getReportOptions().addOption(report);
+ opt.addOption(report);
+
+ Option top = OptionBuilder.withLongOpt(TOP)
+ .hasArg()
+ .withDescription("specify the number of nodes to be listed which has" +
+ " data imbalance.")
+ .create();
+ getReportOptions().addOption(top);
+ opt.addOption(top);
+
+ Option node = OptionBuilder.withLongOpt(NODE)
+ .hasArg()
+ .withDescription("Datanode address, " +
+ "it can be DataNodeID, IP or hostname.")
+ .create();
+ getReportOptions().addOption(node);
+ opt.addOption(node);
+ }
+
+ /**
+ * This function parses all command line arguments and returns the appropriate
+ * values.
+ *
+ * @param argv - Argv from main
+ * @return CommandLine
+ */
+ private CommandLine parseArgs(String[] argv, Options opts)
+ throws org.apache.commons.cli.ParseException {
+ BasicParser parser = new BasicParser();
+ return parser.parse(opts, argv);
+ }
+
+ /**
+ * Dispatches calls to the right command Handler classes.
+ *
+ * @param cmd - CommandLine
+ * @param opts options of command line
+ * @param out the output stream used for printing
+ */
+ private int dispatch(CommandLine cmd, Options opts, final PrintStream out)
+ throws Exception {
+ Command currentCommand = null;
+ if (cmd.hasOption(DiskBalancerCLI.PLAN)) {
+ currentCommand = new PlanCommand(getConf());
+ }
+
+ if (cmd.hasOption(DiskBalancerCLI.EXECUTE)) {
+ currentCommand = new ExecuteCommand(getConf());
+ }
+
+ if (cmd.hasOption(DiskBalancerCLI.QUERY)) {
+ currentCommand = new QueryCommand(getConf());
+ }
+
+ if (cmd.hasOption(DiskBalancerCLI.CANCEL)) {
+ currentCommand = new CancelCommand(getConf());
+ }
+
+ if (cmd.hasOption(DiskBalancerCLI.REPORT)) {
+ currentCommand = new ReportCommand(getConf(), out);
+ }
+
+ if (cmd.hasOption(DiskBalancerCLI.HELP)) {
+ currentCommand = new HelpCommand(getConf());
+ }
+
+ // Invoke main help here.
+ if (currentCommand == null) {
+ new HelpCommand(getConf()).execute(null);
+ return 1;
+ }
+
+ currentCommand.execute(cmd);
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35c5943b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
index 7d659af..451ca04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -41,18 +41,19 @@ import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector;
import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ConnectorFactory;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode;
+import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.Lists;
-import static org.apache.hadoop.hdfs.tools.DiskBalancer.CANCEL;
-import static org.apache.hadoop.hdfs.tools.DiskBalancer.HELP;
-import static org.apache.hadoop.hdfs.tools.DiskBalancer.NODE;
-import static org.apache.hadoop.hdfs.tools.DiskBalancer.PLAN;
-import static org.apache.hadoop.hdfs.tools.DiskBalancer.QUERY;
-import static org.apache.hadoop.hdfs.tools.DiskBalancer.REPORT;
+import static org.apache.hadoop.hdfs.tools.DiskBalancerCLI.CANCEL;
+import static org.apache.hadoop.hdfs.tools.DiskBalancerCLI.HELP;
+import static org.apache.hadoop.hdfs.tools.DiskBalancerCLI.NODE;
+import static org.apache.hadoop.hdfs.tools.DiskBalancerCLI.PLAN;
+import static org.apache.hadoop.hdfs.tools.DiskBalancerCLI.QUERY;
+import static org.apache.hadoop.hdfs.tools.DiskBalancerCLI.REPORT;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
@@ -387,8 +388,7 @@ public class TestDiskBalancerCommand {
private List<String> runCommandInternal(final String cmdLine) throws
Exception {
String[] cmds = StringUtils.split(cmdLine, ' ');
- org.apache.hadoop.hdfs.tools.DiskBalancer db =
- new org.apache.hadoop.hdfs.tools.DiskBalancer(conf);
+ DiskBalancerCLI db = new DiskBalancerCLI(conf);
ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
PrintStream out = new PrintStream(bufOut);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/43] hadoop git commit: HADOOP-13465. Design Server.Call to be
extensible for unified call queue. Contributed by Daryn Sharp.
Posted by ae...@apache.org.
HADOOP-13465. Design Server.Call to be extensible for unified call queue. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76cd81f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76cd81f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76cd81f4
Branch: refs/heads/HDFS-7240
Commit: 76cd81f4b656f0d40a4b2122e15f04ea53d8020b
Parents: 3069df7
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Sep 1 15:44:44 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Sep 1 15:44:44 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 336 +++++++++++--------
1 file changed, 191 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76cd81f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 4c73f6a..80b3303 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -354,10 +354,9 @@ public abstract class Server {
*/
public static InetAddress getRemoteIp() {
Call call = CurCall.get();
- return (call != null && call.connection != null) ? call.connection
- .getHostInetAddress() : null;
+ return (call != null ) ? call.getHostInetAddress() : null;
}
-
+
/**
* Returns the clientId from the current RPC request
*/
@@ -380,10 +379,9 @@ public abstract class Server {
*/
public static UserGroupInformation getRemoteUser() {
Call call = CurCall.get();
- return (call != null && call.connection != null) ? call.connection.user
- : null;
+ return (call != null) ? call.getRemoteUser() : null;
}
-
+
/** Return true if the invocation was through an RPC.
*/
public static boolean isRpcInvocation() {
@@ -483,7 +481,7 @@ public abstract class Server {
if ((rpcMetrics.getProcessingSampleCount() > minSampleSize) &&
(processingTime > threeSigma)) {
if(LOG.isWarnEnabled()) {
- String client = CurCall.get().connection.toString();
+ String client = CurCall.get().toString();
LOG.warn(
"Slow RPC : " + methodName + " took " + processingTime +
" milliseconds to process from client " + client);
@@ -657,62 +655,65 @@ public abstract class Server {
CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT);
}
- /** A call queued for handling. */
- public static class Call implements Schedulable {
- private final int callId; // the client's call id
- private final int retryCount; // the retry count of the call
- private final Writable rpcRequest; // Serialized Rpc request from client
- private final Connection connection; // connection to client
- private long timestamp; // time received when response is null
- // time served when response is not null
- private ByteBuffer rpcResponse; // the response for this call
+ /** A generic call queued for handling. */
+ public static class Call implements Schedulable,
+ PrivilegedExceptionAction<Void> {
+ final int callId; // the client's call id
+ final int retryCount; // the retry count of the call
+ long timestamp; // time received when response is null
+ // time served when response is not null
private AtomicInteger responseWaitCount = new AtomicInteger(1);
- private final RPC.RpcKind rpcKind;
- private final byte[] clientId;
+ final RPC.RpcKind rpcKind;
+ final byte[] clientId;
private final TraceScope traceScope; // the HTrace scope on the server side
private final CallerContext callerContext; // the call context
private int priorityLevel;
// the priority level assigned by scheduler, 0 by default
- private Call(Call call) {
- this(call.callId, call.retryCount, call.rpcRequest, call.connection,
- call.rpcKind, call.clientId, call.traceScope, call.callerContext);
+ Call(Call call) {
+ this(call.callId, call.retryCount, call.rpcKind, call.clientId,
+ call.traceScope, call.callerContext);
}
- public Call(int id, int retryCount, Writable param,
- Connection connection) {
- this(id, retryCount, param, connection, RPC.RpcKind.RPC_BUILTIN,
- RpcConstants.DUMMY_CLIENT_ID);
+ Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId) {
+ this(id, retryCount, kind, clientId, null, null);
}
- public Call(int id, int retryCount, Writable param, Connection connection,
+ @VisibleForTesting // primarily TestNamenodeRetryCache
+ public Call(int id, int retryCount, Void ignore1, Void ignore2,
RPC.RpcKind kind, byte[] clientId) {
- this(id, retryCount, param, connection, kind, clientId, null, null);
+ this(id, retryCount, kind, clientId, null, null);
}
- public Call(int id, int retryCount, Writable param, Connection connection,
- RPC.RpcKind kind, byte[] clientId, TraceScope traceScope,
- CallerContext callerContext) {
+ Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId,
+ TraceScope traceScope, CallerContext callerContext) {
this.callId = id;
this.retryCount = retryCount;
- this.rpcRequest = param;
- this.connection = connection;
this.timestamp = Time.now();
- this.rpcResponse = null;
this.rpcKind = kind;
this.clientId = clientId;
this.traceScope = traceScope;
this.callerContext = callerContext;
}
-
+
@Override
public String toString() {
- return rpcRequest + " from " + connection + " Call#" + callId + " Retry#"
- + retryCount;
+ return "Call#" + callId + " Retry#" + retryCount;
}
- public void setResponse(ByteBuffer response) {
- this.rpcResponse = response;
+ public Void run() throws Exception {
+ return null;
+ }
+ // should eventually be abstract but need to avoid breaking tests
+ public UserGroupInformation getRemoteUser() {
+ return null;
+ }
+ public InetAddress getHostInetAddress() {
+ return null;
+ }
+ public String getHostAddress() {
+ InetAddress addr = getHostInetAddress();
+ return (addr != null) ? addr.getHostAddress() : null;
}
/**
@@ -724,34 +725,36 @@ public abstract class Server {
*/
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void postponeResponse() {
+ public final void postponeResponse() {
int count = responseWaitCount.incrementAndGet();
assert count > 0 : "response has already been sent";
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void sendResponse() throws IOException {
+ public final void sendResponse() throws IOException {
int count = responseWaitCount.decrementAndGet();
assert count >= 0 : "response has already been sent";
if (count == 0) {
- connection.sendResponse(this);
+ doResponse(null);
}
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void abortResponse(Throwable t) throws IOException {
+ public final void abortResponse(Throwable t) throws IOException {
// don't send response if the call was already sent or aborted.
if (responseWaitCount.getAndSet(-1) > 0) {
- connection.abortResponse(this, t);
+ doResponse(t);
}
}
+ void doResponse(Throwable t) throws IOException {}
+
// For Schedulable
@Override
public UserGroupInformation getUserGroupInformation() {
- return connection.user;
+ return getRemoteUser();
}
@Override
@@ -764,6 +767,114 @@ public abstract class Server {
}
}
+ /** A RPC extended call queued for handling. */
+ private class RpcCall extends Call {
+ final Connection connection; // connection to client
+ final Writable rpcRequest; // Serialized Rpc request from client
+ ByteBuffer rpcResponse; // the response for this call
+
+ RpcCall(RpcCall call) {
+ super(call);
+ this.connection = call.connection;
+ this.rpcRequest = call.rpcRequest;
+ }
+
+ RpcCall(Connection connection, int id) {
+ this(connection, id, RpcConstants.INVALID_RETRY_COUNT);
+ }
+
+ RpcCall(Connection connection, int id, int retryCount) {
+ this(connection, id, retryCount, null,
+ RPC.RpcKind.RPC_BUILTIN, RpcConstants.DUMMY_CLIENT_ID,
+ null, null);
+ }
+
+ RpcCall(Connection connection, int id, int retryCount,
+ Writable param, RPC.RpcKind kind, byte[] clientId,
+ TraceScope traceScope, CallerContext context) {
+ super(id, retryCount, kind, clientId, traceScope, context);
+ this.connection = connection;
+ this.rpcRequest = param;
+ }
+
+ @Override
+ public UserGroupInformation getRemoteUser() {
+ return connection.user;
+ }
+
+ @Override
+ public InetAddress getHostInetAddress() {
+ return connection.getHostInetAddress();
+ }
+
+ @Override
+ public Void run() throws Exception {
+ if (!connection.channel.isOpen()) {
+ Server.LOG.info(Thread.currentThread().getName() + ": skipped " + this);
+ return null;
+ }
+ String errorClass = null;
+ String error = null;
+ RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
+ RpcErrorCodeProto detailedErr = null;
+ Writable value = null;
+
+ try {
+ value = call(
+ rpcKind, connection.protocolName, rpcRequest, timestamp);
+ } catch (Throwable e) {
+ if (e instanceof UndeclaredThrowableException) {
+ e = e.getCause();
+ }
+ logException(Server.LOG, e, this);
+ if (e instanceof RpcServerException) {
+ RpcServerException rse = ((RpcServerException)e);
+ returnStatus = rse.getRpcStatusProto();
+ detailedErr = rse.getRpcErrorCodeProto();
+ } else {
+ returnStatus = RpcStatusProto.ERROR;
+ detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
+ }
+ errorClass = e.getClass().getName();
+ error = StringUtils.stringifyException(e);
+ // Remove redundant error class name from the beginning of the
+ // stack trace
+ String exceptionHdr = errorClass + ": ";
+ if (error.startsWith(exceptionHdr)) {
+ error = error.substring(exceptionHdr.length());
+ }
+ }
+ setupResponse(this, returnStatus, detailedErr,
+ value, errorClass, error);
+ sendResponse();
+ return null;
+ }
+
+ void setResponse(ByteBuffer response) throws IOException {
+ this.rpcResponse = response;
+ }
+
+ @Override
+ void doResponse(Throwable t) throws IOException {
+ RpcCall call = this;
+ if (t != null) {
+ // clone the call to prevent a race with another thread stomping
+ // on the response while being sent. the original call is
+ // effectively discarded since the wait count won't hit zero
+ call = new RpcCall(this);
+ setupResponse(call,
+ RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
+ null, t.getClass().getName(), StringUtils.stringifyException(t));
+ }
+ connection.sendResponse(call);
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " " + rpcRequest + " from " + connection;
+ }
+ }
+
/** Listens on the socket. Creates jobs for the handler threads*/
private class Listener extends Thread {
@@ -1094,22 +1205,22 @@ public abstract class Server {
if(LOG.isDebugEnabled()) {
LOG.debug("Checking for old call responses.");
}
- ArrayList<Call> calls;
+ ArrayList<RpcCall> calls;
// get the list of channels from list of keys.
synchronized (writeSelector.keys()) {
- calls = new ArrayList<Call>(writeSelector.keys().size());
+ calls = new ArrayList<RpcCall>(writeSelector.keys().size());
iter = writeSelector.keys().iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
- Call call = (Call)key.attachment();
+ RpcCall call = (RpcCall)key.attachment();
if (call != null && key.channel() == call.connection.channel) {
calls.add(call);
}
}
}
-
- for(Call call : calls) {
+
+ for (RpcCall call : calls) {
doPurge(call, now);
}
} catch (OutOfMemoryError e) {
@@ -1127,7 +1238,7 @@ public abstract class Server {
}
private void doAsyncWrite(SelectionKey key) throws IOException {
- Call call = (Call)key.attachment();
+ RpcCall call = (RpcCall)key.attachment();
if (call == null) {
return;
}
@@ -1155,10 +1266,10 @@ public abstract class Server {
// Remove calls that have been pending in the responseQueue
// for a long time.
//
- private void doPurge(Call call, long now) {
- LinkedList<Call> responseQueue = call.connection.responseQueue;
+ private void doPurge(RpcCall call, long now) {
+ LinkedList<RpcCall> responseQueue = call.connection.responseQueue;
synchronized (responseQueue) {
- Iterator<Call> iter = responseQueue.listIterator(0);
+ Iterator<RpcCall> iter = responseQueue.listIterator(0);
while (iter.hasNext()) {
call = iter.next();
if (now > call.timestamp + PURGE_INTERVAL) {
@@ -1172,12 +1283,12 @@ public abstract class Server {
// Processes one response. Returns true if there are no more pending
// data for this channel.
//
- private boolean processResponse(LinkedList<Call> responseQueue,
+ private boolean processResponse(LinkedList<RpcCall> responseQueue,
boolean inHandler) throws IOException {
boolean error = true;
boolean done = false; // there is more data for this channel.
int numElements = 0;
- Call call = null;
+ RpcCall call = null;
try {
synchronized (responseQueue) {
//
@@ -1260,7 +1371,7 @@ public abstract class Server {
//
// Enqueue a response from the application.
//
- void doRespond(Call call) throws IOException {
+ void doRespond(RpcCall call) throws IOException {
synchronized (call.connection.responseQueue) {
// must only wrap before adding to the responseQueue to prevent
// postponed responses from being encrypted and sent out of order.
@@ -1358,7 +1469,7 @@ public abstract class Server {
private SocketChannel channel;
private ByteBuffer data;
private ByteBuffer dataLengthBuffer;
- private LinkedList<Call> responseQueue;
+ private LinkedList<RpcCall> responseQueue;
// number of outstanding rpcs
private AtomicInteger rpcCount = new AtomicInteger();
private long lastContact;
@@ -1385,8 +1496,8 @@ public abstract class Server {
public UserGroupInformation attemptingUser = null; // user name before auth
// Fake 'call' for failed authorization response
- private final Call authFailedCall = new Call(AUTHORIZATION_FAILED_CALL_ID,
- RpcConstants.INVALID_RETRY_COUNT, null, this);
+ private final RpcCall authFailedCall =
+ new RpcCall(this, AUTHORIZATION_FAILED_CALL_ID);
private boolean sentNegotiate = false;
private boolean useWrap = false;
@@ -1409,7 +1520,7 @@ public abstract class Server {
this.hostAddress = addr.getHostAddress();
}
this.remotePort = socket.getPort();
- this.responseQueue = new LinkedList<Call>();
+ this.responseQueue = new LinkedList<RpcCall>();
if (socketSendBufferSize != 0) {
try {
socket.setSendBufferSize(socketSendBufferSize);
@@ -1704,8 +1815,7 @@ public abstract class Server {
}
private void doSaslReply(Message message) throws IOException {
- final Call saslCall = new Call(AuthProtocol.SASL.callId,
- RpcConstants.INVALID_RETRY_COUNT, null, this);
+ final RpcCall saslCall = new RpcCall(this, AuthProtocol.SASL.callId);
setupResponse(saslCall,
RpcStatusProto.SUCCESS, null,
RpcWritable.wrap(message), null, null);
@@ -1922,23 +2032,20 @@ public abstract class Server {
if (clientVersion >= 9) {
// Versions >>9 understand the normal response
- Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, -1);
setupResponse(fakeCall,
RpcStatusProto.FATAL, RpcErrorCodeProto.FATAL_VERSION_MISMATCH,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion >= 3) {
- Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, -1);
// Versions 3 to 8 use older response
setupResponseOldVersionFatal(buffer, fakeCall,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion == 2) { // Hadoop 0.18.3
- Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, 0);
DataOutputStream out = new DataOutputStream(buffer);
out.writeInt(0); // call ID
out.writeBoolean(true); // error
@@ -1950,7 +2057,7 @@ public abstract class Server {
}
private void setupHttpRequestOnIpcPortResponse() throws IOException {
- Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null, this);
+ RpcCall fakeCall = new RpcCall(this, 0);
fakeCall.setResponse(ByteBuffer.wrap(
RECEIVED_HTTP_REQ_RESPONSE.getBytes(StandardCharsets.UTF_8)));
fakeCall.sendResponse();
@@ -2098,7 +2205,7 @@ public abstract class Server {
}
} catch (WrappedRpcServerException wrse) { // inform client of error
Throwable ioe = wrse.getCause();
- final Call call = new Call(callId, retry, null, this);
+ final RpcCall call = new RpcCall(this, callId, retry);
setupResponse(call,
RpcStatusProto.FATAL, wrse.getRpcErrorCodeProto(), null,
ioe.getClass().getName(), ioe.getMessage());
@@ -2198,8 +2305,9 @@ public abstract class Server {
.build();
}
- Call call = new Call(header.getCallId(), header.getRetryCount(),
- rpcRequest, this, ProtoUtil.convert(header.getRpcKind()),
+ RpcCall call = new RpcCall(this, header.getCallId(),
+ header.getRetryCount(), rpcRequest,
+ ProtoUtil.convert(header.getRpcKind()),
header.getClientId().toByteArray(), traceScope, callerContext);
// Save the priority level assignment by the scheduler
@@ -2323,21 +2431,10 @@ public abstract class Server {
}
}
- private void sendResponse(Call call) throws IOException {
+ private void sendResponse(RpcCall call) throws IOException {
responder.doRespond(call);
}
- private void abortResponse(Call call, Throwable t) throws IOException {
- // clone the call to prevent a race with the other thread stomping
- // on the response while being sent. the original call is
- // effectively discarded since the wait count won't hit zero
- call = new Call(call);
- setupResponse(call,
- RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
- null, t.getClass().getName(), StringUtils.stringifyException(t));
- call.sendResponse();
- }
-
/**
* Get service class for connection
* @return the serviceClass
@@ -2388,16 +2485,6 @@ public abstract class Server {
if (LOG.isDebugEnabled()) {
LOG.debug(Thread.currentThread().getName() + ": " + call + " for RpcKind " + call.rpcKind);
}
- if (!call.connection.channel.isOpen()) {
- LOG.info(Thread.currentThread().getName() + ": skipped " + call);
- continue;
- }
- String errorClass = null;
- String error = null;
- RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
- RpcErrorCodeProto detailedErr = null;
- Writable value = null;
-
CurCall.set(call);
if (call.traceScope != null) {
call.traceScope.reattach();
@@ -2406,53 +2493,11 @@ public abstract class Server {
}
// always update the current call context
CallerContext.setCurrent(call.callerContext);
-
- try {
- // Make the call as the user via Subject.doAs, thus associating
- // the call with the Subject
- if (call.connection.user == null) {
- value = call(call.rpcKind, call.connection.protocolName, call.rpcRequest,
- call.timestamp);
- } else {
- value =
- call.connection.user.doAs
- (new PrivilegedExceptionAction<Writable>() {
- @Override
- public Writable run() throws Exception {
- // make the call
- return call(call.rpcKind, call.connection.protocolName,
- call.rpcRequest, call.timestamp);
-
- }
- }
- );
- }
- } catch (Throwable e) {
- if (e instanceof UndeclaredThrowableException) {
- e = e.getCause();
- }
- logException(LOG, e, call);
- if (e instanceof RpcServerException) {
- RpcServerException rse = ((RpcServerException)e);
- returnStatus = rse.getRpcStatusProto();
- detailedErr = rse.getRpcErrorCodeProto();
- } else {
- returnStatus = RpcStatusProto.ERROR;
- detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
- }
- errorClass = e.getClass().getName();
- error = StringUtils.stringifyException(e);
- // Remove redundant error class name from the beginning of the stack trace
- String exceptionHdr = errorClass + ": ";
- if (error.startsWith(exceptionHdr)) {
- error = error.substring(exceptionHdr.length());
- }
- }
- CurCall.set(null);
- synchronized (call.connection.responseQueue) {
- setupResponse(call, returnStatus, detailedErr,
- value, errorClass, error);
- call.sendResponse();
+ UserGroupInformation remoteUser = call.getRemoteUser();
+ if (remoteUser != null) {
+ remoteUser.doAs(call);
+ } else {
+ call.run();
}
} catch (InterruptedException e) {
if (running) { // unexpected -- log it
@@ -2469,6 +2514,7 @@ public abstract class Server {
StringUtils.stringifyException(e));
}
} finally {
+ CurCall.set(null);
IOUtils.cleanup(LOG, traceScope);
}
}
@@ -2670,7 +2716,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponse(
- Call call, RpcStatusProto status, RpcErrorCodeProto erCode,
+ RpcCall call, RpcStatusProto status, RpcErrorCodeProto erCode,
Writable rv, String errorClass, String error)
throws IOException {
RpcResponseHeaderProto.Builder headerBuilder =
@@ -2704,7 +2750,7 @@ public abstract class Server {
}
}
- private void setupResponse(Call call,
+ private void setupResponse(RpcCall call,
RpcResponseHeaderProto header, Writable rv) throws IOException {
ResponseBuffer buf = responseBuffer.get().reset();
try {
@@ -2738,7 +2784,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponseOldVersionFatal(ByteArrayOutputStream response,
- Call call,
+ RpcCall call,
Writable rv, String errorClass, String error)
throws IOException {
final int OLD_VERSION_FATAL_STATUS = -1;
@@ -2751,7 +2797,7 @@ public abstract class Server {
call.setResponse(ByteBuffer.wrap(response.toByteArray()));
}
- private void wrapWithSasl(Call call) throws IOException {
+ private void wrapWithSasl(RpcCall call) throws IOException {
if (call.connection.saslServer != null) {
byte[] token = call.rpcResponse.array();
// synchronization may be needed since there can be multiple Handler
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/43] hadoop git commit: FsPermission string constructor does not
recognize sticky bit. Contributed by Atul Sikaria
Posted by ae...@apache.org.
FsPermission string constructor does not recognize sticky bit. Contributed by Atul Sikaria
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3069df75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3069df75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3069df75
Branch: refs/heads/HDFS-7240
Commit: 3069df75513180a5b56130a7e05c6e339ab0a66a
Parents: dcd21d0
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Sep 1 11:32:26 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Sep 1 11:32:26 2016 -0700
----------------------------------------------------------------------
.../hadoop/fs/permission/FsPermission.java | 2 +-
.../apache/hadoop/fs/permission/RawParser.java | 44 +
.../hadoop/fs/permission/TestFsPermission.java | 1061 +++++++++---------
3 files changed, 585 insertions(+), 522 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3069df75/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index b535fd6..78255bbd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -103,7 +103,7 @@ public class FsPermission implements Writable {
* @throws IllegalArgumentException if <code>mode</code> is invalid
*/
public FsPermission(String mode) {
- this(new UmaskParser(mode).getUMask());
+ this(new RawParser(mode).getPermission());
}
/** Return user {@link FsAction}. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3069df75/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java
new file mode 100644
index 0000000..3bbe9cb
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.permission;
+
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class RawParser extends PermissionParser {
+ private static Pattern rawOctalPattern =
+ Pattern.compile("^\\s*([01]?)([0-7]{3})\\s*$");
+ private static Pattern rawNormalPattern =
+ Pattern.compile("\\G\\s*([ugoa]*)([+=-]+)([rwxt]*)([,\\s]*)\\s*");
+
+ private short permission;
+
+ public RawParser(String modeStr) throws IllegalArgumentException {
+ super(modeStr, rawNormalPattern, rawOctalPattern);
+ permission = (short)combineModes(0, false);
+ }
+
+ public short getPermission() {
+ return permission;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3069df75/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
index 04dbe01..6368a57 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
@@ -56,6 +56,8 @@ public class TestFsPermission extends TestCase {
public void testConvertingPermissions() {
for(short s = 0; s <= 01777; s++) {
assertEquals(s, new FsPermission(s).toShort());
+ // check string formats
+ assertEquals(s, new FsPermission(String.format("%03o", s)).toShort());
}
short s = 0;
@@ -75,6 +77,7 @@ public class TestFsPermission extends TestCase {
}
}
}
+ assertEquals(02000, s);
}
public void testSpecialBitsToString() {
@@ -129,6 +132,20 @@ public class TestFsPermission extends TestCase {
}
}
+ public void testSymbolicPermission() {
+ for (int i = 0; i < SYMBOLIC.length; ++i) {
+ short val = 0777;
+ val &= ~Short.valueOf(SYMBOLIC[i][1], 8);
+ assertEquals(val, new FsPermission(SYMBOLIC[i][0]).toShort());
+ }
+ // add sticky bit to "other" when exec enabled
+ for (int i = 1; i < SYMBOLIC.length; i += 2) {
+ short val = 01777;
+ val &= ~Short.valueOf(SYMBOLIC[i][1], 8);
+ assertEquals(val, new FsPermission(SYMBOLIC[i][0] + "t").toShort());
+ }
+ }
+
public void testUMaskParser() throws IOException {
Configuration conf = new Configuration();
@@ -150,529 +167,13 @@ public class TestFsPermission extends TestCase {
Configuration conf = new Configuration();
// Test some symbolic to octal settings
- // Symbolic umask list is generated in linux shell using by the command:
- // umask 0; umask <octal number>; umask -S
- String [][] symbolic = new String [][] {
- {"a+rw", "111",},
- {"u=rwx,g=rwx,o=rwx", "0",},
- {"u=rwx,g=rwx,o=rw", "1",},
- {"u=rwx,g=rwx,o=rx", "2",},
- {"u=rwx,g=rwx,o=r", "3",},
- {"u=rwx,g=rwx,o=wx", "4",},
- {"u=rwx,g=rwx,o=w", "5",},
- {"u=rwx,g=rwx,o=x", "6",},
- {"u=rwx,g=rwx,o=", "7",},
- {"u=rwx,g=rw,o=rwx", "10",},
- {"u=rwx,g=rw,o=rw", "11",},
- {"u=rwx,g=rw,o=rx", "12",},
- {"u=rwx,g=rw,o=r", "13",},
- {"u=rwx,g=rw,o=wx", "14",},
- {"u=rwx,g=rw,o=w", "15",},
- {"u=rwx,g=rw,o=x", "16",},
- {"u=rwx,g=rw,o=", "17",},
- {"u=rwx,g=rx,o=rwx", "20",},
- {"u=rwx,g=rx,o=rw", "21",},
- {"u=rwx,g=rx,o=rx", "22",},
- {"u=rwx,g=rx,o=r", "23",},
- {"u=rwx,g=rx,o=wx", "24",},
- {"u=rwx,g=rx,o=w", "25",},
- {"u=rwx,g=rx,o=x", "26",},
- {"u=rwx,g=rx,o=", "27",},
- {"u=rwx,g=r,o=rwx", "30",},
- {"u=rwx,g=r,o=rw", "31",},
- {"u=rwx,g=r,o=rx", "32",},
- {"u=rwx,g=r,o=r", "33",},
- {"u=rwx,g=r,o=wx", "34",},
- {"u=rwx,g=r,o=w", "35",},
- {"u=rwx,g=r,o=x", "36",},
- {"u=rwx,g=r,o=", "37",},
- {"u=rwx,g=wx,o=rwx", "40",},
- {"u=rwx,g=wx,o=rw", "41",},
- {"u=rwx,g=wx,o=rx", "42",},
- {"u=rwx,g=wx,o=r", "43",},
- {"u=rwx,g=wx,o=wx", "44",},
- {"u=rwx,g=wx,o=w", "45",},
- {"u=rwx,g=wx,o=x", "46",},
- {"u=rwx,g=wx,o=", "47",},
- {"u=rwx,g=w,o=rwx", "50",},
- {"u=rwx,g=w,o=rw", "51",},
- {"u=rwx,g=w,o=rx", "52",},
- {"u=rwx,g=w,o=r", "53",},
- {"u=rwx,g=w,o=wx", "54",},
- {"u=rwx,g=w,o=w", "55",},
- {"u=rwx,g=w,o=x", "56",},
- {"u=rwx,g=w,o=", "57",},
- {"u=rwx,g=x,o=rwx", "60",},
- {"u=rwx,g=x,o=rw", "61",},
- {"u=rwx,g=x,o=rx", "62",},
- {"u=rwx,g=x,o=r", "63",},
- {"u=rwx,g=x,o=wx", "64",},
- {"u=rwx,g=x,o=w", "65",},
- {"u=rwx,g=x,o=x", "66",},
- {"u=rwx,g=x,o=", "67",},
- {"u=rwx,g=,o=rwx", "70",},
- {"u=rwx,g=,o=rw", "71",},
- {"u=rwx,g=,o=rx", "72",},
- {"u=rwx,g=,o=r", "73",},
- {"u=rwx,g=,o=wx", "74",},
- {"u=rwx,g=,o=w", "75",},
- {"u=rwx,g=,o=x", "76",},
- {"u=rwx,g=,o=", "77",},
- {"u=rw,g=rwx,o=rwx", "100",},
- {"u=rw,g=rwx,o=rw", "101",},
- {"u=rw,g=rwx,o=rx", "102",},
- {"u=rw,g=rwx,o=r", "103",},
- {"u=rw,g=rwx,o=wx", "104",},
- {"u=rw,g=rwx,o=w", "105",},
- {"u=rw,g=rwx,o=x", "106",},
- {"u=rw,g=rwx,o=", "107",},
- {"u=rw,g=rw,o=rwx", "110",},
- {"u=rw,g=rw,o=rw", "111",},
- {"u=rw,g=rw,o=rx", "112",},
- {"u=rw,g=rw,o=r", "113",},
- {"u=rw,g=rw,o=wx", "114",},
- {"u=rw,g=rw,o=w", "115",},
- {"u=rw,g=rw,o=x", "116",},
- {"u=rw,g=rw,o=", "117",},
- {"u=rw,g=rx,o=rwx", "120",},
- {"u=rw,g=rx,o=rw", "121",},
- {"u=rw,g=rx,o=rx", "122",},
- {"u=rw,g=rx,o=r", "123",},
- {"u=rw,g=rx,o=wx", "124",},
- {"u=rw,g=rx,o=w", "125",},
- {"u=rw,g=rx,o=x", "126",},
- {"u=rw,g=rx,o=", "127",},
- {"u=rw,g=r,o=rwx", "130",},
- {"u=rw,g=r,o=rw", "131",},
- {"u=rw,g=r,o=rx", "132",},
- {"u=rw,g=r,o=r", "133",},
- {"u=rw,g=r,o=wx", "134",},
- {"u=rw,g=r,o=w", "135",},
- {"u=rw,g=r,o=x", "136",},
- {"u=rw,g=r,o=", "137",},
- {"u=rw,g=wx,o=rwx", "140",},
- {"u=rw,g=wx,o=rw", "141",},
- {"u=rw,g=wx,o=rx", "142",},
- {"u=rw,g=wx,o=r", "143",},
- {"u=rw,g=wx,o=wx", "144",},
- {"u=rw,g=wx,o=w", "145",},
- {"u=rw,g=wx,o=x", "146",},
- {"u=rw,g=wx,o=", "147",},
- {"u=rw,g=w,o=rwx", "150",},
- {"u=rw,g=w,o=rw", "151",},
- {"u=rw,g=w,o=rx", "152",},
- {"u=rw,g=w,o=r", "153",},
- {"u=rw,g=w,o=wx", "154",},
- {"u=rw,g=w,o=w", "155",},
- {"u=rw,g=w,o=x", "156",},
- {"u=rw,g=w,o=", "157",},
- {"u=rw,g=x,o=rwx", "160",},
- {"u=rw,g=x,o=rw", "161",},
- {"u=rw,g=x,o=rx", "162",},
- {"u=rw,g=x,o=r", "163",},
- {"u=rw,g=x,o=wx", "164",},
- {"u=rw,g=x,o=w", "165",},
- {"u=rw,g=x,o=x", "166",},
- {"u=rw,g=x,o=", "167",},
- {"u=rw,g=,o=rwx", "170",},
- {"u=rw,g=,o=rw", "171",},
- {"u=rw,g=,o=rx", "172",},
- {"u=rw,g=,o=r", "173",},
- {"u=rw,g=,o=wx", "174",},
- {"u=rw,g=,o=w", "175",},
- {"u=rw,g=,o=x", "176",},
- {"u=rw,g=,o=", "177",},
- {"u=rx,g=rwx,o=rwx", "200",},
- {"u=rx,g=rwx,o=rw", "201",},
- {"u=rx,g=rwx,o=rx", "202",},
- {"u=rx,g=rwx,o=r", "203",},
- {"u=rx,g=rwx,o=wx", "204",},
- {"u=rx,g=rwx,o=w", "205",},
- {"u=rx,g=rwx,o=x", "206",},
- {"u=rx,g=rwx,o=", "207",},
- {"u=rx,g=rw,o=rwx", "210",},
- {"u=rx,g=rw,o=rw", "211",},
- {"u=rx,g=rw,o=rx", "212",},
- {"u=rx,g=rw,o=r", "213",},
- {"u=rx,g=rw,o=wx", "214",},
- {"u=rx,g=rw,o=w", "215",},
- {"u=rx,g=rw,o=x", "216",},
- {"u=rx,g=rw,o=", "217",},
- {"u=rx,g=rx,o=rwx", "220",},
- {"u=rx,g=rx,o=rw", "221",},
- {"u=rx,g=rx,o=rx", "222",},
- {"u=rx,g=rx,o=r", "223",},
- {"u=rx,g=rx,o=wx", "224",},
- {"u=rx,g=rx,o=w", "225",},
- {"u=rx,g=rx,o=x", "226",},
- {"u=rx,g=rx,o=", "227",},
- {"u=rx,g=r,o=rwx", "230",},
- {"u=rx,g=r,o=rw", "231",},
- {"u=rx,g=r,o=rx", "232",},
- {"u=rx,g=r,o=r", "233",},
- {"u=rx,g=r,o=wx", "234",},
- {"u=rx,g=r,o=w", "235",},
- {"u=rx,g=r,o=x", "236",},
- {"u=rx,g=r,o=", "237",},
- {"u=rx,g=wx,o=rwx", "240",},
- {"u=rx,g=wx,o=rw", "241",},
- {"u=rx,g=wx,o=rx", "242",},
- {"u=rx,g=wx,o=r", "243",},
- {"u=rx,g=wx,o=wx", "244",},
- {"u=rx,g=wx,o=w", "245",},
- {"u=rx,g=wx,o=x", "246",},
- {"u=rx,g=wx,o=", "247",},
- {"u=rx,g=w,o=rwx", "250",},
- {"u=rx,g=w,o=rw", "251",},
- {"u=rx,g=w,o=rx", "252",},
- {"u=rx,g=w,o=r", "253",},
- {"u=rx,g=w,o=wx", "254",},
- {"u=rx,g=w,o=w", "255",},
- {"u=rx,g=w,o=x", "256",},
- {"u=rx,g=w,o=", "257",},
- {"u=rx,g=x,o=rwx", "260",},
- {"u=rx,g=x,o=rw", "261",},
- {"u=rx,g=x,o=rx", "262",},
- {"u=rx,g=x,o=r", "263",},
- {"u=rx,g=x,o=wx", "264",},
- {"u=rx,g=x,o=w", "265",},
- {"u=rx,g=x,o=x", "266",},
- {"u=rx,g=x,o=", "267",},
- {"u=rx,g=,o=rwx", "270",},
- {"u=rx,g=,o=rw", "271",},
- {"u=rx,g=,o=rx", "272",},
- {"u=rx,g=,o=r", "273",},
- {"u=rx,g=,o=wx", "274",},
- {"u=rx,g=,o=w", "275",},
- {"u=rx,g=,o=x", "276",},
- {"u=rx,g=,o=", "277",},
- {"u=r,g=rwx,o=rwx", "300",},
- {"u=r,g=rwx,o=rw", "301",},
- {"u=r,g=rwx,o=rx", "302",},
- {"u=r,g=rwx,o=r", "303",},
- {"u=r,g=rwx,o=wx", "304",},
- {"u=r,g=rwx,o=w", "305",},
- {"u=r,g=rwx,o=x", "306",},
- {"u=r,g=rwx,o=", "307",},
- {"u=r,g=rw,o=rwx", "310",},
- {"u=r,g=rw,o=rw", "311",},
- {"u=r,g=rw,o=rx", "312",},
- {"u=r,g=rw,o=r", "313",},
- {"u=r,g=rw,o=wx", "314",},
- {"u=r,g=rw,o=w", "315",},
- {"u=r,g=rw,o=x", "316",},
- {"u=r,g=rw,o=", "317",},
- {"u=r,g=rx,o=rwx", "320",},
- {"u=r,g=rx,o=rw", "321",},
- {"u=r,g=rx,o=rx", "322",},
- {"u=r,g=rx,o=r", "323",},
- {"u=r,g=rx,o=wx", "324",},
- {"u=r,g=rx,o=w", "325",},
- {"u=r,g=rx,o=x", "326",},
- {"u=r,g=rx,o=", "327",},
- {"u=r,g=r,o=rwx", "330",},
- {"u=r,g=r,o=rw", "331",},
- {"u=r,g=r,o=rx", "332",},
- {"u=r,g=r,o=r", "333",},
- {"u=r,g=r,o=wx", "334",},
- {"u=r,g=r,o=w", "335",},
- {"u=r,g=r,o=x", "336",},
- {"u=r,g=r,o=", "337",},
- {"u=r,g=wx,o=rwx", "340",},
- {"u=r,g=wx,o=rw", "341",},
- {"u=r,g=wx,o=rx", "342",},
- {"u=r,g=wx,o=r", "343",},
- {"u=r,g=wx,o=wx", "344",},
- {"u=r,g=wx,o=w", "345",},
- {"u=r,g=wx,o=x", "346",},
- {"u=r,g=wx,o=", "347",},
- {"u=r,g=w,o=rwx", "350",},
- {"u=r,g=w,o=rw", "351",},
- {"u=r,g=w,o=rx", "352",},
- {"u=r,g=w,o=r", "353",},
- {"u=r,g=w,o=wx", "354",},
- {"u=r,g=w,o=w", "355",},
- {"u=r,g=w,o=x", "356",},
- {"u=r,g=w,o=", "357",},
- {"u=r,g=x,o=rwx", "360",},
- {"u=r,g=x,o=rw", "361",},
- {"u=r,g=x,o=rx", "362",},
- {"u=r,g=x,o=r", "363",},
- {"u=r,g=x,o=wx", "364",},
- {"u=r,g=x,o=w", "365",},
- {"u=r,g=x,o=x", "366",},
- {"u=r,g=x,o=", "367",},
- {"u=r,g=,o=rwx", "370",},
- {"u=r,g=,o=rw", "371",},
- {"u=r,g=,o=rx", "372",},
- {"u=r,g=,o=r", "373",},
- {"u=r,g=,o=wx", "374",},
- {"u=r,g=,o=w", "375",},
- {"u=r,g=,o=x", "376",},
- {"u=r,g=,o=", "377",},
- {"u=wx,g=rwx,o=rwx", "400",},
- {"u=wx,g=rwx,o=rw", "401",},
- {"u=wx,g=rwx,o=rx", "402",},
- {"u=wx,g=rwx,o=r", "403",},
- {"u=wx,g=rwx,o=wx", "404",},
- {"u=wx,g=rwx,o=w", "405",},
- {"u=wx,g=rwx,o=x", "406",},
- {"u=wx,g=rwx,o=", "407",},
- {"u=wx,g=rw,o=rwx", "410",},
- {"u=wx,g=rw,o=rw", "411",},
- {"u=wx,g=rw,o=rx", "412",},
- {"u=wx,g=rw,o=r", "413",},
- {"u=wx,g=rw,o=wx", "414",},
- {"u=wx,g=rw,o=w", "415",},
- {"u=wx,g=rw,o=x", "416",},
- {"u=wx,g=rw,o=", "417",},
- {"u=wx,g=rx,o=rwx", "420",},
- {"u=wx,g=rx,o=rw", "421",},
- {"u=wx,g=rx,o=rx", "422",},
- {"u=wx,g=rx,o=r", "423",},
- {"u=wx,g=rx,o=wx", "424",},
- {"u=wx,g=rx,o=w", "425",},
- {"u=wx,g=rx,o=x", "426",},
- {"u=wx,g=rx,o=", "427",},
- {"u=wx,g=r,o=rwx", "430",},
- {"u=wx,g=r,o=rw", "431",},
- {"u=wx,g=r,o=rx", "432",},
- {"u=wx,g=r,o=r", "433",},
- {"u=wx,g=r,o=wx", "434",},
- {"u=wx,g=r,o=w", "435",},
- {"u=wx,g=r,o=x", "436",},
- {"u=wx,g=r,o=", "437",},
- {"u=wx,g=wx,o=rwx", "440",},
- {"u=wx,g=wx,o=rw", "441",},
- {"u=wx,g=wx,o=rx", "442",},
- {"u=wx,g=wx,o=r", "443",},
- {"u=wx,g=wx,o=wx", "444",},
- {"u=wx,g=wx,o=w", "445",},
- {"u=wx,g=wx,o=x", "446",},
- {"u=wx,g=wx,o=", "447",},
- {"u=wx,g=w,o=rwx", "450",},
- {"u=wx,g=w,o=rw", "451",},
- {"u=wx,g=w,o=rx", "452",},
- {"u=wx,g=w,o=r", "453",},
- {"u=wx,g=w,o=wx", "454",},
- {"u=wx,g=w,o=w", "455",},
- {"u=wx,g=w,o=x", "456",},
- {"u=wx,g=w,o=", "457",},
- {"u=wx,g=x,o=rwx", "460",},
- {"u=wx,g=x,o=rw", "461",},
- {"u=wx,g=x,o=rx", "462",},
- {"u=wx,g=x,o=r", "463",},
- {"u=wx,g=x,o=wx", "464",},
- {"u=wx,g=x,o=w", "465",},
- {"u=wx,g=x,o=x", "466",},
- {"u=wx,g=x,o=", "467",},
- {"u=wx,g=,o=rwx", "470",},
- {"u=wx,g=,o=rw", "471",},
- {"u=wx,g=,o=rx", "472",},
- {"u=wx,g=,o=r", "473",},
- {"u=wx,g=,o=wx", "474",},
- {"u=wx,g=,o=w", "475",},
- {"u=wx,g=,o=x", "476",},
- {"u=wx,g=,o=", "477",},
- {"u=w,g=rwx,o=rwx", "500",},
- {"u=w,g=rwx,o=rw", "501",},
- {"u=w,g=rwx,o=rx", "502",},
- {"u=w,g=rwx,o=r", "503",},
- {"u=w,g=rwx,o=wx", "504",},
- {"u=w,g=rwx,o=w", "505",},
- {"u=w,g=rwx,o=x", "506",},
- {"u=w,g=rwx,o=", "507",},
- {"u=w,g=rw,o=rwx", "510",},
- {"u=w,g=rw,o=rw", "511",},
- {"u=w,g=rw,o=rx", "512",},
- {"u=w,g=rw,o=r", "513",},
- {"u=w,g=rw,o=wx", "514",},
- {"u=w,g=rw,o=w", "515",},
- {"u=w,g=rw,o=x", "516",},
- {"u=w,g=rw,o=", "517",},
- {"u=w,g=rx,o=rwx", "520",},
- {"u=w,g=rx,o=rw", "521",},
- {"u=w,g=rx,o=rx", "522",},
- {"u=w,g=rx,o=r", "523",},
- {"u=w,g=rx,o=wx", "524",},
- {"u=w,g=rx,o=w", "525",},
- {"u=w,g=rx,o=x", "526",},
- {"u=w,g=rx,o=", "527",},
- {"u=w,g=r,o=rwx", "530",},
- {"u=w,g=r,o=rw", "531",},
- {"u=w,g=r,o=rx", "532",},
- {"u=w,g=r,o=r", "533",},
- {"u=w,g=r,o=wx", "534",},
- {"u=w,g=r,o=w", "535",},
- {"u=w,g=r,o=x", "536",},
- {"u=w,g=r,o=", "537",},
- {"u=w,g=wx,o=rwx", "540",},
- {"u=w,g=wx,o=rw", "541",},
- {"u=w,g=wx,o=rx", "542",},
- {"u=w,g=wx,o=r", "543",},
- {"u=w,g=wx,o=wx", "544",},
- {"u=w,g=wx,o=w", "545",},
- {"u=w,g=wx,o=x", "546",},
- {"u=w,g=wx,o=", "547",},
- {"u=w,g=w,o=rwx", "550",},
- {"u=w,g=w,o=rw", "551",},
- {"u=w,g=w,o=rx", "552",},
- {"u=w,g=w,o=r", "553",},
- {"u=w,g=w,o=wx", "554",},
- {"u=w,g=w,o=w", "555",},
- {"u=w,g=w,o=x", "556",},
- {"u=w,g=w,o=", "557",},
- {"u=w,g=x,o=rwx", "560",},
- {"u=w,g=x,o=rw", "561",},
- {"u=w,g=x,o=rx", "562",},
- {"u=w,g=x,o=r", "563",},
- {"u=w,g=x,o=wx", "564",},
- {"u=w,g=x,o=w", "565",},
- {"u=w,g=x,o=x", "566",},
- {"u=w,g=x,o=", "567",},
- {"u=w,g=,o=rwx", "570",},
- {"u=w,g=,o=rw", "571",},
- {"u=w,g=,o=rx", "572",},
- {"u=w,g=,o=r", "573",},
- {"u=w,g=,o=wx", "574",},
- {"u=w,g=,o=w", "575",},
- {"u=w,g=,o=x", "576",},
- {"u=w,g=,o=", "577",},
- {"u=x,g=rwx,o=rwx", "600",},
- {"u=x,g=rwx,o=rw", "601",},
- {"u=x,g=rwx,o=rx", "602",},
- {"u=x,g=rwx,o=r", "603",},
- {"u=x,g=rwx,o=wx", "604",},
- {"u=x,g=rwx,o=w", "605",},
- {"u=x,g=rwx,o=x", "606",},
- {"u=x,g=rwx,o=", "607",},
- {"u=x,g=rw,o=rwx", "610",},
- {"u=x,g=rw,o=rw", "611",},
- {"u=x,g=rw,o=rx", "612",},
- {"u=x,g=rw,o=r", "613",},
- {"u=x,g=rw,o=wx", "614",},
- {"u=x,g=rw,o=w", "615",},
- {"u=x,g=rw,o=x", "616",},
- {"u=x,g=rw,o=", "617",},
- {"u=x,g=rx,o=rwx", "620",},
- {"u=x,g=rx,o=rw", "621",},
- {"u=x,g=rx,o=rx", "622",},
- {"u=x,g=rx,o=r", "623",},
- {"u=x,g=rx,o=wx", "624",},
- {"u=x,g=rx,o=w", "625",},
- {"u=x,g=rx,o=x", "626",},
- {"u=x,g=rx,o=", "627",},
- {"u=x,g=r,o=rwx", "630",},
- {"u=x,g=r,o=rw", "631",},
- {"u=x,g=r,o=rx", "632",},
- {"u=x,g=r,o=r", "633",},
- {"u=x,g=r,o=wx", "634",},
- {"u=x,g=r,o=w", "635",},
- {"u=x,g=r,o=x", "636",},
- {"u=x,g=r,o=", "637",},
- {"u=x,g=wx,o=rwx", "640",},
- {"u=x,g=wx,o=rw", "641",},
- {"u=x,g=wx,o=rx", "642",},
- {"u=x,g=wx,o=r", "643",},
- {"u=x,g=wx,o=wx", "644",},
- {"u=x,g=wx,o=w", "645",},
- {"u=x,g=wx,o=x", "646",},
- {"u=x,g=wx,o=", "647",},
- {"u=x,g=w,o=rwx", "650",},
- {"u=x,g=w,o=rw", "651",},
- {"u=x,g=w,o=rx", "652",},
- {"u=x,g=w,o=r", "653",},
- {"u=x,g=w,o=wx", "654",},
- {"u=x,g=w,o=w", "655",},
- {"u=x,g=w,o=x", "656",},
- {"u=x,g=w,o=", "657",},
- {"u=x,g=x,o=rwx", "660",},
- {"u=x,g=x,o=rw", "661",},
- {"u=x,g=x,o=rx", "662",},
- {"u=x,g=x,o=r", "663",},
- {"u=x,g=x,o=wx", "664",},
- {"u=x,g=x,o=w", "665",},
- {"u=x,g=x,o=x", "666",},
- {"u=x,g=x,o=", "667",},
- {"u=x,g=,o=rwx", "670",},
- {"u=x,g=,o=rw", "671",},
- {"u=x,g=,o=rx", "672",},
- {"u=x,g=,o=r", "673",},
- {"u=x,g=,o=wx", "674",},
- {"u=x,g=,o=w", "675",},
- {"u=x,g=,o=x", "676",},
- {"u=x,g=,o=", "677",},
- {"u=,g=rwx,o=rwx", "700",},
- {"u=,g=rwx,o=rw", "701",},
- {"u=,g=rwx,o=rx", "702",},
- {"u=,g=rwx,o=r", "703",},
- {"u=,g=rwx,o=wx", "704",},
- {"u=,g=rwx,o=w", "705",},
- {"u=,g=rwx,o=x", "706",},
- {"u=,g=rwx,o=", "707",},
- {"u=,g=rw,o=rwx", "710",},
- {"u=,g=rw,o=rw", "711",},
- {"u=,g=rw,o=rx", "712",},
- {"u=,g=rw,o=r", "713",},
- {"u=,g=rw,o=wx", "714",},
- {"u=,g=rw,o=w", "715",},
- {"u=,g=rw,o=x", "716",},
- {"u=,g=rw,o=", "717",},
- {"u=,g=rx,o=rwx", "720",},
- {"u=,g=rx,o=rw", "721",},
- {"u=,g=rx,o=rx", "722",},
- {"u=,g=rx,o=r", "723",},
- {"u=,g=rx,o=wx", "724",},
- {"u=,g=rx,o=w", "725",},
- {"u=,g=rx,o=x", "726",},
- {"u=,g=rx,o=", "727",},
- {"u=,g=r,o=rwx", "730",},
- {"u=,g=r,o=rw", "731",},
- {"u=,g=r,o=rx", "732",},
- {"u=,g=r,o=r", "733",},
- {"u=,g=r,o=wx", "734",},
- {"u=,g=r,o=w", "735",},
- {"u=,g=r,o=x", "736",},
- {"u=,g=r,o=", "737",},
- {"u=,g=wx,o=rwx", "740",},
- {"u=,g=wx,o=rw", "741",},
- {"u=,g=wx,o=rx", "742",},
- {"u=,g=wx,o=r", "743",},
- {"u=,g=wx,o=wx", "744",},
- {"u=,g=wx,o=w", "745",},
- {"u=,g=wx,o=x", "746",},
- {"u=,g=wx,o=", "747",},
- {"u=,g=w,o=rwx", "750",},
- {"u=,g=w,o=rw", "751",},
- {"u=,g=w,o=rx", "752",},
- {"u=,g=w,o=r", "753",},
- {"u=,g=w,o=wx", "754",},
- {"u=,g=w,o=w", "755",},
- {"u=,g=w,o=x", "756",},
- {"u=,g=w,o=", "757",},
- {"u=,g=x,o=rwx", "760",},
- {"u=,g=x,o=rw", "761",},
- {"u=,g=x,o=rx", "762",},
- {"u=,g=x,o=r", "763",},
- {"u=,g=x,o=wx", "764",},
- {"u=,g=x,o=w", "765",},
- {"u=,g=x,o=x", "766",},
- {"u=,g=x,o=", "767",},
- {"u=,g=,o=rwx", "770",},
- {"u=,g=,o=rw", "771",},
- {"u=,g=,o=rx", "772",},
- {"u=,g=,o=r", "773",},
- {"u=,g=,o=wx", "774",},
- {"u=,g=,o=w", "775",},
- {"u=,g=,o=x", "776",},
- {"u=,g=,o=", "777"}
- };
-
- for(int i = 0; i < symbolic.length; i += 2) {
- conf.set(FsPermission.UMASK_LABEL, symbolic[i][0]);
- short val = Short.valueOf(symbolic[i][1], 8);
+ for (int i = 0; i < SYMBOLIC.length; ++i) {
+ conf.set(FsPermission.UMASK_LABEL, SYMBOLIC[i][0]);
+ short val = Short.valueOf(SYMBOLIC[i][1], 8);
assertEquals(val, FsPermission.getUMask(conf).toShort());
}
+ conf.set(FsPermission.UMASK_LABEL, "a+rw");
+ assertEquals(0111, FsPermission.getUMask(conf).toShort());
}
public void testBadUmasks() {
@@ -695,4 +196,522 @@ public class TestFsPermission extends TestCase {
msg.contains(umask) &&
msg.contains("octal or symbolic");
}
+
+ // Symbolic umask list is generated in linux shell using by the command:
+ // umask 0; umask <octal number>; umask -S
+ static final String[][] SYMBOLIC = new String[][] {
+ {"u=rwx,g=rwx,o=rwx", "0"},
+ {"u=rwx,g=rwx,o=rw", "1"},
+ {"u=rwx,g=rwx,o=rx", "2"},
+ {"u=rwx,g=rwx,o=r", "3"},
+ {"u=rwx,g=rwx,o=wx", "4"},
+ {"u=rwx,g=rwx,o=w", "5"},
+ {"u=rwx,g=rwx,o=x", "6"},
+ {"u=rwx,g=rwx,o=", "7"},
+ {"u=rwx,g=rw,o=rwx", "10"},
+ {"u=rwx,g=rw,o=rw", "11"},
+ {"u=rwx,g=rw,o=rx", "12"},
+ {"u=rwx,g=rw,o=r", "13"},
+ {"u=rwx,g=rw,o=wx", "14"},
+ {"u=rwx,g=rw,o=w", "15"},
+ {"u=rwx,g=rw,o=x", "16"},
+ {"u=rwx,g=rw,o=", "17"},
+ {"u=rwx,g=rx,o=rwx", "20"},
+ {"u=rwx,g=rx,o=rw", "21"},
+ {"u=rwx,g=rx,o=rx", "22"},
+ {"u=rwx,g=rx,o=r", "23"},
+ {"u=rwx,g=rx,o=wx", "24"},
+ {"u=rwx,g=rx,o=w", "25"},
+ {"u=rwx,g=rx,o=x", "26"},
+ {"u=rwx,g=rx,o=", "27"},
+ {"u=rwx,g=r,o=rwx", "30"},
+ {"u=rwx,g=r,o=rw", "31"},
+ {"u=rwx,g=r,o=rx", "32"},
+ {"u=rwx,g=r,o=r", "33"},
+ {"u=rwx,g=r,o=wx", "34"},
+ {"u=rwx,g=r,o=w", "35"},
+ {"u=rwx,g=r,o=x", "36"},
+ {"u=rwx,g=r,o=", "37"},
+ {"u=rwx,g=wx,o=rwx", "40"},
+ {"u=rwx,g=wx,o=rw", "41"},
+ {"u=rwx,g=wx,o=rx", "42"},
+ {"u=rwx,g=wx,o=r", "43"},
+ {"u=rwx,g=wx,o=wx", "44"},
+ {"u=rwx,g=wx,o=w", "45"},
+ {"u=rwx,g=wx,o=x", "46"},
+ {"u=rwx,g=wx,o=", "47"},
+ {"u=rwx,g=w,o=rwx", "50"},
+ {"u=rwx,g=w,o=rw", "51"},
+ {"u=rwx,g=w,o=rx", "52"},
+ {"u=rwx,g=w,o=r", "53"},
+ {"u=rwx,g=w,o=wx", "54"},
+ {"u=rwx,g=w,o=w", "55"},
+ {"u=rwx,g=w,o=x", "56"},
+ {"u=rwx,g=w,o=", "57"},
+ {"u=rwx,g=x,o=rwx", "60"},
+ {"u=rwx,g=x,o=rw", "61"},
+ {"u=rwx,g=x,o=rx", "62"},
+ {"u=rwx,g=x,o=r", "63"},
+ {"u=rwx,g=x,o=wx", "64"},
+ {"u=rwx,g=x,o=w", "65"},
+ {"u=rwx,g=x,o=x", "66"},
+ {"u=rwx,g=x,o=", "67"},
+ {"u=rwx,g=,o=rwx", "70"},
+ {"u=rwx,g=,o=rw", "71"},
+ {"u=rwx,g=,o=rx", "72"},
+ {"u=rwx,g=,o=r", "73"},
+ {"u=rwx,g=,o=wx", "74"},
+ {"u=rwx,g=,o=w", "75"},
+ {"u=rwx,g=,o=x", "76"},
+ {"u=rwx,g=,o=", "77"},
+ {"u=rw,g=rwx,o=rwx", "100"},
+ {"u=rw,g=rwx,o=rw", "101"},
+ {"u=rw,g=rwx,o=rx", "102"},
+ {"u=rw,g=rwx,o=r", "103"},
+ {"u=rw,g=rwx,o=wx", "104"},
+ {"u=rw,g=rwx,o=w", "105"},
+ {"u=rw,g=rwx,o=x", "106"},
+ {"u=rw,g=rwx,o=", "107"},
+ {"u=rw,g=rw,o=rwx", "110"},
+ {"u=rw,g=rw,o=rw", "111"},
+ {"u=rw,g=rw,o=rx", "112"},
+ {"u=rw,g=rw,o=r", "113"},
+ {"u=rw,g=rw,o=wx", "114"},
+ {"u=rw,g=rw,o=w", "115"},
+ {"u=rw,g=rw,o=x", "116"},
+ {"u=rw,g=rw,o=", "117"},
+ {"u=rw,g=rx,o=rwx", "120"},
+ {"u=rw,g=rx,o=rw", "121"},
+ {"u=rw,g=rx,o=rx", "122"},
+ {"u=rw,g=rx,o=r", "123"},
+ {"u=rw,g=rx,o=wx", "124"},
+ {"u=rw,g=rx,o=w", "125"},
+ {"u=rw,g=rx,o=x", "126"},
+ {"u=rw,g=rx,o=", "127"},
+ {"u=rw,g=r,o=rwx", "130"},
+ {"u=rw,g=r,o=rw", "131"},
+ {"u=rw,g=r,o=rx", "132"},
+ {"u=rw,g=r,o=r", "133"},
+ {"u=rw,g=r,o=wx", "134"},
+ {"u=rw,g=r,o=w", "135"},
+ {"u=rw,g=r,o=x", "136"},
+ {"u=rw,g=r,o=", "137"},
+ {"u=rw,g=wx,o=rwx", "140"},
+ {"u=rw,g=wx,o=rw", "141"},
+ {"u=rw,g=wx,o=rx", "142"},
+ {"u=rw,g=wx,o=r", "143"},
+ {"u=rw,g=wx,o=wx", "144"},
+ {"u=rw,g=wx,o=w", "145"},
+ {"u=rw,g=wx,o=x", "146"},
+ {"u=rw,g=wx,o=", "147"},
+ {"u=rw,g=w,o=rwx", "150"},
+ {"u=rw,g=w,o=rw", "151"},
+ {"u=rw,g=w,o=rx", "152"},
+ {"u=rw,g=w,o=r", "153"},
+ {"u=rw,g=w,o=wx", "154"},
+ {"u=rw,g=w,o=w", "155"},
+ {"u=rw,g=w,o=x", "156"},
+ {"u=rw,g=w,o=", "157"},
+ {"u=rw,g=x,o=rwx", "160"},
+ {"u=rw,g=x,o=rw", "161"},
+ {"u=rw,g=x,o=rx", "162"},
+ {"u=rw,g=x,o=r", "163"},
+ {"u=rw,g=x,o=wx", "164"},
+ {"u=rw,g=x,o=w", "165"},
+ {"u=rw,g=x,o=x", "166"},
+ {"u=rw,g=x,o=", "167"},
+ {"u=rw,g=,o=rwx", "170"},
+ {"u=rw,g=,o=rw", "171"},
+ {"u=rw,g=,o=rx", "172"},
+ {"u=rw,g=,o=r", "173"},
+ {"u=rw,g=,o=wx", "174"},
+ {"u=rw,g=,o=w", "175"},
+ {"u=rw,g=,o=x", "176"},
+ {"u=rw,g=,o=", "177"},
+ {"u=rx,g=rwx,o=rwx", "200"},
+ {"u=rx,g=rwx,o=rw", "201"},
+ {"u=rx,g=rwx,o=rx", "202"},
+ {"u=rx,g=rwx,o=r", "203"},
+ {"u=rx,g=rwx,o=wx", "204"},
+ {"u=rx,g=rwx,o=w", "205"},
+ {"u=rx,g=rwx,o=x", "206"},
+ {"u=rx,g=rwx,o=", "207"},
+ {"u=rx,g=rw,o=rwx", "210"},
+ {"u=rx,g=rw,o=rw", "211"},
+ {"u=rx,g=rw,o=rx", "212"},
+ {"u=rx,g=rw,o=r", "213"},
+ {"u=rx,g=rw,o=wx", "214"},
+ {"u=rx,g=rw,o=w", "215"},
+ {"u=rx,g=rw,o=x", "216"},
+ {"u=rx,g=rw,o=", "217"},
+ {"u=rx,g=rx,o=rwx", "220"},
+ {"u=rx,g=rx,o=rw", "221"},
+ {"u=rx,g=rx,o=rx", "222"},
+ {"u=rx,g=rx,o=r", "223"},
+ {"u=rx,g=rx,o=wx", "224"},
+ {"u=rx,g=rx,o=w", "225"},
+ {"u=rx,g=rx,o=x", "226"},
+ {"u=rx,g=rx,o=", "227"},
+ {"u=rx,g=r,o=rwx", "230"},
+ {"u=rx,g=r,o=rw", "231"},
+ {"u=rx,g=r,o=rx", "232"},
+ {"u=rx,g=r,o=r", "233"},
+ {"u=rx,g=r,o=wx", "234"},
+ {"u=rx,g=r,o=w", "235"},
+ {"u=rx,g=r,o=x", "236"},
+ {"u=rx,g=r,o=", "237"},
+ {"u=rx,g=wx,o=rwx", "240"},
+ {"u=rx,g=wx,o=rw", "241"},
+ {"u=rx,g=wx,o=rx", "242"},
+ {"u=rx,g=wx,o=r", "243"},
+ {"u=rx,g=wx,o=wx", "244"},
+ {"u=rx,g=wx,o=w", "245"},
+ {"u=rx,g=wx,o=x", "246"},
+ {"u=rx,g=wx,o=", "247"},
+ {"u=rx,g=w,o=rwx", "250"},
+ {"u=rx,g=w,o=rw", "251"},
+ {"u=rx,g=w,o=rx", "252"},
+ {"u=rx,g=w,o=r", "253"},
+ {"u=rx,g=w,o=wx", "254"},
+ {"u=rx,g=w,o=w", "255"},
+ {"u=rx,g=w,o=x", "256"},
+ {"u=rx,g=w,o=", "257"},
+ {"u=rx,g=x,o=rwx", "260"},
+ {"u=rx,g=x,o=rw", "261"},
+ {"u=rx,g=x,o=rx", "262"},
+ {"u=rx,g=x,o=r", "263"},
+ {"u=rx,g=x,o=wx", "264"},
+ {"u=rx,g=x,o=w", "265"},
+ {"u=rx,g=x,o=x", "266"},
+ {"u=rx,g=x,o=", "267"},
+ {"u=rx,g=,o=rwx", "270"},
+ {"u=rx,g=,o=rw", "271"},
+ {"u=rx,g=,o=rx", "272"},
+ {"u=rx,g=,o=r", "273"},
+ {"u=rx,g=,o=wx", "274"},
+ {"u=rx,g=,o=w", "275"},
+ {"u=rx,g=,o=x", "276"},
+ {"u=rx,g=,o=", "277"},
+ {"u=r,g=rwx,o=rwx", "300"},
+ {"u=r,g=rwx,o=rw", "301"},
+ {"u=r,g=rwx,o=rx", "302"},
+ {"u=r,g=rwx,o=r", "303"},
+ {"u=r,g=rwx,o=wx", "304"},
+ {"u=r,g=rwx,o=w", "305"},
+ {"u=r,g=rwx,o=x", "306"},
+ {"u=r,g=rwx,o=", "307"},
+ {"u=r,g=rw,o=rwx", "310"},
+ {"u=r,g=rw,o=rw", "311"},
+ {"u=r,g=rw,o=rx", "312"},
+ {"u=r,g=rw,o=r", "313"},
+ {"u=r,g=rw,o=wx", "314"},
+ {"u=r,g=rw,o=w", "315"},
+ {"u=r,g=rw,o=x", "316"},
+ {"u=r,g=rw,o=", "317"},
+ {"u=r,g=rx,o=rwx", "320"},
+ {"u=r,g=rx,o=rw", "321"},
+ {"u=r,g=rx,o=rx", "322"},
+ {"u=r,g=rx,o=r", "323"},
+ {"u=r,g=rx,o=wx", "324"},
+ {"u=r,g=rx,o=w", "325"},
+ {"u=r,g=rx,o=x", "326"},
+ {"u=r,g=rx,o=", "327"},
+ {"u=r,g=r,o=rwx", "330"},
+ {"u=r,g=r,o=rw", "331"},
+ {"u=r,g=r,o=rx", "332"},
+ {"u=r,g=r,o=r", "333"},
+ {"u=r,g=r,o=wx", "334"},
+ {"u=r,g=r,o=w", "335"},
+ {"u=r,g=r,o=x", "336"},
+ {"u=r,g=r,o=", "337"},
+ {"u=r,g=wx,o=rwx", "340"},
+ {"u=r,g=wx,o=rw", "341"},
+ {"u=r,g=wx,o=rx", "342"},
+ {"u=r,g=wx,o=r", "343"},
+ {"u=r,g=wx,o=wx", "344"},
+ {"u=r,g=wx,o=w", "345"},
+ {"u=r,g=wx,o=x", "346"},
+ {"u=r,g=wx,o=", "347"},
+ {"u=r,g=w,o=rwx", "350"},
+ {"u=r,g=w,o=rw", "351"},
+ {"u=r,g=w,o=rx", "352"},
+ {"u=r,g=w,o=r", "353"},
+ {"u=r,g=w,o=wx", "354"},
+ {"u=r,g=w,o=w", "355"},
+ {"u=r,g=w,o=x", "356"},
+ {"u=r,g=w,o=", "357"},
+ {"u=r,g=x,o=rwx", "360"},
+ {"u=r,g=x,o=rw", "361"},
+ {"u=r,g=x,o=rx", "362"},
+ {"u=r,g=x,o=r", "363"},
+ {"u=r,g=x,o=wx", "364"},
+ {"u=r,g=x,o=w", "365"},
+ {"u=r,g=x,o=x", "366"},
+ {"u=r,g=x,o=", "367"},
+ {"u=r,g=,o=rwx", "370"},
+ {"u=r,g=,o=rw", "371"},
+ {"u=r,g=,o=rx", "372"},
+ {"u=r,g=,o=r", "373"},
+ {"u=r,g=,o=wx", "374"},
+ {"u=r,g=,o=w", "375"},
+ {"u=r,g=,o=x", "376"},
+ {"u=r,g=,o=", "377"},
+ {"u=wx,g=rwx,o=rwx", "400"},
+ {"u=wx,g=rwx,o=rw", "401"},
+ {"u=wx,g=rwx,o=rx", "402"},
+ {"u=wx,g=rwx,o=r", "403"},
+ {"u=wx,g=rwx,o=wx", "404"},
+ {"u=wx,g=rwx,o=w", "405"},
+ {"u=wx,g=rwx,o=x", "406"},
+ {"u=wx,g=rwx,o=", "407"},
+ {"u=wx,g=rw,o=rwx", "410"},
+ {"u=wx,g=rw,o=rw", "411"},
+ {"u=wx,g=rw,o=rx", "412"},
+ {"u=wx,g=rw,o=r", "413"},
+ {"u=wx,g=rw,o=wx", "414"},
+ {"u=wx,g=rw,o=w", "415"},
+ {"u=wx,g=rw,o=x", "416"},
+ {"u=wx,g=rw,o=", "417"},
+ {"u=wx,g=rx,o=rwx", "420"},
+ {"u=wx,g=rx,o=rw", "421"},
+ {"u=wx,g=rx,o=rx", "422"},
+ {"u=wx,g=rx,o=r", "423"},
+ {"u=wx,g=rx,o=wx", "424"},
+ {"u=wx,g=rx,o=w", "425"},
+ {"u=wx,g=rx,o=x", "426"},
+ {"u=wx,g=rx,o=", "427"},
+ {"u=wx,g=r,o=rwx", "430"},
+ {"u=wx,g=r,o=rw", "431"},
+ {"u=wx,g=r,o=rx", "432"},
+ {"u=wx,g=r,o=r", "433"},
+ {"u=wx,g=r,o=wx", "434"},
+ {"u=wx,g=r,o=w", "435"},
+ {"u=wx,g=r,o=x", "436"},
+ {"u=wx,g=r,o=", "437"},
+ {"u=wx,g=wx,o=rwx", "440"},
+ {"u=wx,g=wx,o=rw", "441"},
+ {"u=wx,g=wx,o=rx", "442"},
+ {"u=wx,g=wx,o=r", "443"},
+ {"u=wx,g=wx,o=wx", "444"},
+ {"u=wx,g=wx,o=w", "445"},
+ {"u=wx,g=wx,o=x", "446"},
+ {"u=wx,g=wx,o=", "447"},
+ {"u=wx,g=w,o=rwx", "450"},
+ {"u=wx,g=w,o=rw", "451"},
+ {"u=wx,g=w,o=rx", "452"},
+ {"u=wx,g=w,o=r", "453"},
+ {"u=wx,g=w,o=wx", "454"},
+ {"u=wx,g=w,o=w", "455"},
+ {"u=wx,g=w,o=x", "456"},
+ {"u=wx,g=w,o=", "457"},
+ {"u=wx,g=x,o=rwx", "460"},
+ {"u=wx,g=x,o=rw", "461"},
+ {"u=wx,g=x,o=rx", "462"},
+ {"u=wx,g=x,o=r", "463"},
+ {"u=wx,g=x,o=wx", "464"},
+ {"u=wx,g=x,o=w", "465"},
+ {"u=wx,g=x,o=x", "466"},
+ {"u=wx,g=x,o=", "467"},
+ {"u=wx,g=,o=rwx", "470"},
+ {"u=wx,g=,o=rw", "471"},
+ {"u=wx,g=,o=rx", "472"},
+ {"u=wx,g=,o=r", "473"},
+ {"u=wx,g=,o=wx", "474"},
+ {"u=wx,g=,o=w", "475"},
+ {"u=wx,g=,o=x", "476"},
+ {"u=wx,g=,o=", "477"},
+ {"u=w,g=rwx,o=rwx", "500"},
+ {"u=w,g=rwx,o=rw", "501"},
+ {"u=w,g=rwx,o=rx", "502"},
+ {"u=w,g=rwx,o=r", "503"},
+ {"u=w,g=rwx,o=wx", "504"},
+ {"u=w,g=rwx,o=w", "505"},
+ {"u=w,g=rwx,o=x", "506"},
+ {"u=w,g=rwx,o=", "507"},
+ {"u=w,g=rw,o=rwx", "510"},
+ {"u=w,g=rw,o=rw", "511"},
+ {"u=w,g=rw,o=rx", "512"},
+ {"u=w,g=rw,o=r", "513"},
+ {"u=w,g=rw,o=wx", "514"},
+ {"u=w,g=rw,o=w", "515"},
+ {"u=w,g=rw,o=x", "516"},
+ {"u=w,g=rw,o=", "517"},
+ {"u=w,g=rx,o=rwx", "520"},
+ {"u=w,g=rx,o=rw", "521"},
+ {"u=w,g=rx,o=rx", "522"},
+ {"u=w,g=rx,o=r", "523"},
+ {"u=w,g=rx,o=wx", "524"},
+ {"u=w,g=rx,o=w", "525"},
+ {"u=w,g=rx,o=x", "526"},
+ {"u=w,g=rx,o=", "527"},
+ {"u=w,g=r,o=rwx", "530"},
+ {"u=w,g=r,o=rw", "531"},
+ {"u=w,g=r,o=rx", "532"},
+ {"u=w,g=r,o=r", "533"},
+ {"u=w,g=r,o=wx", "534"},
+ {"u=w,g=r,o=w", "535"},
+ {"u=w,g=r,o=x", "536"},
+ {"u=w,g=r,o=", "537"},
+ {"u=w,g=wx,o=rwx", "540"},
+ {"u=w,g=wx,o=rw", "541"},
+ {"u=w,g=wx,o=rx", "542"},
+ {"u=w,g=wx,o=r", "543"},
+ {"u=w,g=wx,o=wx", "544"},
+ {"u=w,g=wx,o=w", "545"},
+ {"u=w,g=wx,o=x", "546"},
+ {"u=w,g=wx,o=", "547"},
+ {"u=w,g=w,o=rwx", "550"},
+ {"u=w,g=w,o=rw", "551"},
+ {"u=w,g=w,o=rx", "552"},
+ {"u=w,g=w,o=r", "553"},
+ {"u=w,g=w,o=wx", "554"},
+ {"u=w,g=w,o=w", "555"},
+ {"u=w,g=w,o=x", "556"},
+ {"u=w,g=w,o=", "557"},
+ {"u=w,g=x,o=rwx", "560"},
+ {"u=w,g=x,o=rw", "561"},
+ {"u=w,g=x,o=rx", "562"},
+ {"u=w,g=x,o=r", "563"},
+ {"u=w,g=x,o=wx", "564"},
+ {"u=w,g=x,o=w", "565"},
+ {"u=w,g=x,o=x", "566"},
+ {"u=w,g=x,o=", "567"},
+ {"u=w,g=,o=rwx", "570"},
+ {"u=w,g=,o=rw", "571"},
+ {"u=w,g=,o=rx", "572"},
+ {"u=w,g=,o=r", "573"},
+ {"u=w,g=,o=wx", "574"},
+ {"u=w,g=,o=w", "575"},
+ {"u=w,g=,o=x", "576"},
+ {"u=w,g=,o=", "577"},
+ {"u=x,g=rwx,o=rwx", "600"},
+ {"u=x,g=rwx,o=rw", "601"},
+ {"u=x,g=rwx,o=rx", "602"},
+ {"u=x,g=rwx,o=r", "603"},
+ {"u=x,g=rwx,o=wx", "604"},
+ {"u=x,g=rwx,o=w", "605"},
+ {"u=x,g=rwx,o=x", "606"},
+ {"u=x,g=rwx,o=", "607"},
+ {"u=x,g=rw,o=rwx", "610"},
+ {"u=x,g=rw,o=rw", "611"},
+ {"u=x,g=rw,o=rx", "612"},
+ {"u=x,g=rw,o=r", "613"},
+ {"u=x,g=rw,o=wx", "614"},
+ {"u=x,g=rw,o=w", "615"},
+ {"u=x,g=rw,o=x", "616"},
+ {"u=x,g=rw,o=", "617"},
+ {"u=x,g=rx,o=rwx", "620"},
+ {"u=x,g=rx,o=rw", "621"},
+ {"u=x,g=rx,o=rx", "622"},
+ {"u=x,g=rx,o=r", "623"},
+ {"u=x,g=rx,o=wx", "624"},
+ {"u=x,g=rx,o=w", "625"},
+ {"u=x,g=rx,o=x", "626"},
+ {"u=x,g=rx,o=", "627"},
+ {"u=x,g=r,o=rwx", "630"},
+ {"u=x,g=r,o=rw", "631"},
+ {"u=x,g=r,o=rx", "632"},
+ {"u=x,g=r,o=r", "633"},
+ {"u=x,g=r,o=wx", "634"},
+ {"u=x,g=r,o=w", "635"},
+ {"u=x,g=r,o=x", "636"},
+ {"u=x,g=r,o=", "637"},
+ {"u=x,g=wx,o=rwx", "640"},
+ {"u=x,g=wx,o=rw", "641"},
+ {"u=x,g=wx,o=rx", "642"},
+ {"u=x,g=wx,o=r", "643"},
+ {"u=x,g=wx,o=wx", "644"},
+ {"u=x,g=wx,o=w", "645"},
+ {"u=x,g=wx,o=x", "646"},
+ {"u=x,g=wx,o=", "647"},
+ {"u=x,g=w,o=rwx", "650"},
+ {"u=x,g=w,o=rw", "651"},
+ {"u=x,g=w,o=rx", "652"},
+ {"u=x,g=w,o=r", "653"},
+ {"u=x,g=w,o=wx", "654"},
+ {"u=x,g=w,o=w", "655"},
+ {"u=x,g=w,o=x", "656"},
+ {"u=x,g=w,o=", "657"},
+ {"u=x,g=x,o=rwx", "660"},
+ {"u=x,g=x,o=rw", "661"},
+ {"u=x,g=x,o=rx", "662"},
+ {"u=x,g=x,o=r", "663"},
+ {"u=x,g=x,o=wx", "664"},
+ {"u=x,g=x,o=w", "665"},
+ {"u=x,g=x,o=x", "666"},
+ {"u=x,g=x,o=", "667"},
+ {"u=x,g=,o=rwx", "670"},
+ {"u=x,g=,o=rw", "671"},
+ {"u=x,g=,o=rx", "672"},
+ {"u=x,g=,o=r", "673"},
+ {"u=x,g=,o=wx", "674"},
+ {"u=x,g=,o=w", "675"},
+ {"u=x,g=,o=x", "676"},
+ {"u=x,g=,o=", "677"},
+ {"u=,g=rwx,o=rwx", "700"},
+ {"u=,g=rwx,o=rw", "701"},
+ {"u=,g=rwx,o=rx", "702"},
+ {"u=,g=rwx,o=r", "703"},
+ {"u=,g=rwx,o=wx", "704"},
+ {"u=,g=rwx,o=w", "705"},
+ {"u=,g=rwx,o=x", "706"},
+ {"u=,g=rwx,o=", "707"},
+ {"u=,g=rw,o=rwx", "710"},
+ {"u=,g=rw,o=rw", "711"},
+ {"u=,g=rw,o=rx", "712"},
+ {"u=,g=rw,o=r", "713"},
+ {"u=,g=rw,o=wx", "714"},
+ {"u=,g=rw,o=w", "715"},
+ {"u=,g=rw,o=x", "716"},
+ {"u=,g=rw,o=", "717"},
+ {"u=,g=rx,o=rwx", "720"},
+ {"u=,g=rx,o=rw", "721"},
+ {"u=,g=rx,o=rx", "722"},
+ {"u=,g=rx,o=r", "723"},
+ {"u=,g=rx,o=wx", "724"},
+ {"u=,g=rx,o=w", "725"},
+ {"u=,g=rx,o=x", "726"},
+ {"u=,g=rx,o=", "727"},
+ {"u=,g=r,o=rwx", "730"},
+ {"u=,g=r,o=rw", "731"},
+ {"u=,g=r,o=rx", "732"},
+ {"u=,g=r,o=r", "733"},
+ {"u=,g=r,o=wx", "734"},
+ {"u=,g=r,o=w", "735"},
+ {"u=,g=r,o=x", "736"},
+ {"u=,g=r,o=", "737"},
+ {"u=,g=wx,o=rwx", "740"},
+ {"u=,g=wx,o=rw", "741"},
+ {"u=,g=wx,o=rx", "742"},
+ {"u=,g=wx,o=r", "743"},
+ {"u=,g=wx,o=wx", "744"},
+ {"u=,g=wx,o=w", "745"},
+ {"u=,g=wx,o=x", "746"},
+ {"u=,g=wx,o=", "747"},
+ {"u=,g=w,o=rwx", "750"},
+ {"u=,g=w,o=rw", "751"},
+ {"u=,g=w,o=rx", "752"},
+ {"u=,g=w,o=r", "753"},
+ {"u=,g=w,o=wx", "754"},
+ {"u=,g=w,o=w", "755"},
+ {"u=,g=w,o=x", "756"},
+ {"u=,g=w,o=", "757"},
+ {"u=,g=x,o=rwx", "760"},
+ {"u=,g=x,o=rw", "761"},
+ {"u=,g=x,o=rx", "762"},
+ {"u=,g=x,o=r", "763"},
+ {"u=,g=x,o=wx", "764"},
+ {"u=,g=x,o=w", "765"},
+ {"u=,g=x,o=x", "766"},
+ {"u=,g=x,o=", "767"},
+ {"u=,g=,o=rwx", "770"},
+ {"u=,g=,o=rw", "771"},
+ {"u=,g=,o=rx", "772"},
+ {"u=,g=,o=r", "773"},
+ {"u=,g=,o=wx", "774"},
+ {"u=,g=,o=w", "775"},
+ {"u=,g=,o=x", "776"},
+ {"u=,g=,o=", "777"}
+ };
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/43] hadoop git commit: HDFS-10835. Fix typos in httpfs.sh.
Contributed by John Zhuge.
Posted by ae...@apache.org.
HDFS-10835. Fix typos in httpfs.sh. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c213c34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c213c34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c213c34
Branch: refs/heads/HDFS-7240
Commit: 7c213c341d047362e6f8b66ab3ffafaf639d8d31
Parents: f0d5382
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 11:44:26 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 11:44:26 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c213c34/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index a6c9f03..64459bf 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -17,15 +17,15 @@ MYNAME="${BASH_SOURCE-$0}"
function hadoop_usage
{
- hadoop_add_subcommand "run" "Start kms in the current window"
+ hadoop_add_subcommand "run" "Start HttpFS in the current window"
hadoop_add_subcommand "run -security" "Start in the current window with security manager"
- hadoop_add_subcommand "start" "Start kms in a separate window"
+ hadoop_add_subcommand "start" "Start HttpFS in a separate window"
hadoop_add_subcommand "start -security" "Start in a separate window with security manager"
hadoop_add_subcommand "status" "Return the LSB compliant status"
- hadoop_add_subcommand "stop" "Stop kms, waiting up to 5 seconds for the process to end"
- hadoop_add_subcommand "top n" "Stop kms, waiting up to n seconds for the process to end"
- hadoop_add_subcommand "stop -force" "Stop kms, wait up to 5 seconds and then use kill -KILL if still running"
- hadoop_add_subcommand "stop n -force" "Stop kms, wait up to n seconds and then use kill -KILL if still running"
+ hadoop_add_subcommand "stop" "Stop HttpFS, waiting up to 5 seconds for the process to end"
+ hadoop_add_subcommand "stop n" "Stop HttpFS, waiting up to n seconds for the process to end"
+ hadoop_add_subcommand "stop -force" "Stop HttpFS, wait up to 5 seconds and then use kill -KILL if still running"
+ hadoop_add_subcommand "stop n -force" "Stop HttpFS, wait up to n seconds and then use kill -KILL if still running"
hadoop_generate_usage "${MYNAME}" false
}
@@ -47,7 +47,7 @@ else
exit 1
fi
-# The Java System property 'httpfs.http.port' it is not used by Kms,
+# The Java System property 'httpfs.http.port' it is not used by HttpFS,
# it is used in Tomcat's server.xml configuration file
#
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/43] hadoop git commit: HDFS-10845. Change defaults in
hdfs-site.xml to match timeunit type. Contributed by Yiqun Lin
Posted by ae...@apache.org.
HDFS-10845. Change defaults in hdfs-site.xml to match timeunit type. Contributed by Yiqun Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6d839a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6d839a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6d839a6
Branch: refs/heads/HDFS-7240
Commit: b6d839a60ceed733bfacb791fc5ed06116720dd0
Parents: cae33118
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Sep 8 13:29:15 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Sep 8 13:29:30 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/hdfs-default.xml | 22 ++++++++++----------
1 file changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6d839a6/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index d427f88..3f78233 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -650,7 +650,7 @@
<property>
<name>dfs.blockreport.initialDelay</name>
- <value>0</value>
+ <value>0s</value>
<description>
Delay for first block report in seconds. Support multiple time unit
suffix(case insensitive), as described in dfs.heartbeat.interval.
@@ -694,7 +694,7 @@
<property>
<name>dfs.datanode.directoryscan.interval</name>
- <value>21600</value>
+ <value>21600s</value>
<description>Interval in seconds for Datanode to scan data directories and
reconcile the difference between blocks in memory and on the disk.
Support multiple time unit suffix(case insensitive), as described
@@ -732,7 +732,7 @@
<property>
<name>dfs.heartbeat.interval</name>
- <value>3</value>
+ <value>3s</value>
<description>
Determines datanode heartbeat interval in seconds.
Can use the following suffix (case insensitive):
@@ -942,7 +942,7 @@
<property>
<name>dfs.namenode.decommission.interval</name>
- <value>30</value>
+ <value>30s</value>
<description>Namenode periodicity in seconds to check if decommission is
complete. Support multiple time unit suffix(case insensitive), as described
in dfs.heartbeat.interval.
@@ -973,7 +973,7 @@
<property>
<name>dfs.namenode.replication.interval</name>
- <value>3</value>
+ <value>3s</value>
<description>The periodicity in seconds with which the namenode computes
replication work for datanodes. Support multiple time unit suffix(case insensitive),
as described in dfs.heartbeat.interval.
@@ -1071,7 +1071,7 @@
<property>
<name>dfs.namenode.checkpoint.period</name>
- <value>3600</value>
+ <value>3600s</value>
<description>
The number of seconds between two periodic checkpoints.
Support multiple time unit suffix(case insensitive), as described
@@ -1090,7 +1090,7 @@
<property>
<name>dfs.namenode.checkpoint.check.period</name>
- <value>60</value>
+ <value>60s</value>
<description>The SecondaryNameNode and CheckpointNode will poll the NameNode
every 'dfs.namenode.checkpoint.check.period' seconds to query the number
of uncheckpointed transactions. Support multiple time unit suffix(case insensitive),
@@ -1433,7 +1433,7 @@
<property>
<name>dfs.client.datanode-restart.timeout</name>
- <value>30</value>
+ <value>30s</value>
<description>
Expert only. The time to wait, in seconds, from reception of an
datanode shutdown notification for quick restart, until declaring
@@ -1502,7 +1502,7 @@
<property>
<name>dfs.ha.log-roll.period</name>
- <value>120</value>
+ <value>120s</value>
<description>
How often, in seconds, the StandbyNode should ask the active to
roll edit logs. Since the StandbyNode only reads from finalized
@@ -1516,7 +1516,7 @@
<property>
<name>dfs.ha.tail-edits.period</name>
- <value>60</value>
+ <value>60s</value>
<description>
How often, in seconds, the StandbyNode should check for new
finalized log segments in the shared edits log.
@@ -2950,7 +2950,7 @@
<property>
<name>dfs.datanode.bp-ready.timeout</name>
- <value>20</value>
+ <value>20s</value>
<description>
The maximum wait time for datanode to be ready before failing the
received request. Setting this to 0 fails requests right away if the
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/43] hadoop git commit: HDFS-10742. Measure lock time in
FsDatasetImpl. Contributed by Chen Liang
Posted by ae...@apache.org.
HDFS-10742. Measure lock time in FsDatasetImpl. Contributed by Chen Liang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/011f3b24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/011f3b24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/011f3b24
Branch: refs/heads/HDFS-7240
Commit: 011f3b24d4bfda505a90ab5b5576916a41f869c5
Parents: 1192781
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Sep 8 17:52:02 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Sep 8 17:53:40 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/util/AutoCloseableLock.java | 28 ++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../apache/hadoop/hdfs/InstrumentedLock.java | 185 +++++++++++++++++++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 10 +-
.../src/main/resources/hdfs-default.xml | 8 +
.../hadoop/hdfs/TestInstrumentedLock.java | 166 +++++++++++++++++
6 files changed, 395 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/011f3b24/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java
index 2aa8578..d920bc6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AutoCloseableLock.java
@@ -17,22 +17,33 @@
*/
package org.apache.hadoop.util;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* This is a wrap class of a ReentrantLock. Extending AutoCloseable
* interface such that the users can use a try-with-resource syntax.
*/
public class AutoCloseableLock implements AutoCloseable {
- private final ReentrantLock lock;
+ private final Lock lock;
/**
* Creates an instance of {@code AutoCloseableLock}, initializes
- * the underlying {@code ReentrantLock} object.
+ * the underlying lock instance with a new {@code ReentrantLock}.
*/
public AutoCloseableLock() {
- this.lock = new ReentrantLock();
+ this(new ReentrantLock());
+ }
+
+ /**
+ * Wrap provided Lock instance.
+ * @param lock Lock instance to wrap in AutoCloseable API.
+ */
+ public AutoCloseableLock(Lock lock) {
+ this.lock = lock;
}
/**
@@ -86,7 +97,7 @@ public class AutoCloseableLock implements AutoCloseable {
/**
* A wrapper method that makes a call to {@code tryLock()} of
- * the underlying {@code ReentrantLock} object.
+ * the underlying {@code Lock} object.
*
* If the lock is not held by another thread, acquires the lock, set the
* hold count to one and returns {@code true}.
@@ -116,7 +127,12 @@ public class AutoCloseableLock implements AutoCloseable {
* @return {@code true} if any thread holds this lock and
* {@code false} otherwise
*/
- public boolean isLocked() {
- return lock.isLocked();
+ @VisibleForTesting
+ boolean isLocked() {
+ if (lock instanceof ReentrantLock) {
+ return ((ReentrantLock)lock).isLocked();
+ }
+ throw new UnsupportedOperationException();
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/011f3b24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 7bdfd44..caf6b60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -419,6 +419,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY =
"dfs.namenode.read-lock-reporting-threshold-ms";
public static final long DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 5000L;
+ // Threshold for how long the lock warnings must be suppressed
+ public static final String DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY =
+ "dfs.lock.suppress.warning.interval";
+ public static final long DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT =
+ 10000; //ms
public static final String DFS_UPGRADE_DOMAIN_FACTOR = "dfs.namenode.upgrade.domain.factor";
public static final int DFS_UPGRADE_DOMAIN_FACTOR_DEFAULT = DFS_REPLICATION_DEFAULT;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/011f3b24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/InstrumentedLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/InstrumentedLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/InstrumentedLock.java
new file mode 100644
index 0000000..6279e95
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/InstrumentedLock.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Timer;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This is a debugging class that can be used by callers to track
+ * whether a specifc lock is being held for too long and periodically
+ * log a warning and stack trace, if so.
+ *
+ * The logged warnings are throttled so that logs are not spammed.
+ *
+ * A new instance of InstrumentedLock can be created for each object
+ * that needs to be instrumented.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class InstrumentedLock implements Lock {
+
+ private final Lock lock;
+ private final Log logger;
+ private final String name;
+ private final Timer clock;
+
+ /** Minimum gap between two lock warnings. */
+ private final long minLoggingGap;
+ /** Threshold for detecting long lock held time. */
+ private final long lockWarningThreshold;
+
+ // Tracking counters for lock statistics.
+ private volatile long lockAcquireTimestamp;
+ private final AtomicLong lastLogTimestamp;
+ private final AtomicLong warningsSuppressed = new AtomicLong(0);
+
+ /**
+ * Create a instrumented lock instance which logs a warning message
+ * when lock held time is above given threshold.
+ *
+ * @param name the identifier of the lock object
+ * @param logger this class does not have its own logger, will log to the
+ * given logger instead
+ * @param minLoggingGapMs the minimum time gap between two log messages,
+ * this is to avoid spamming to many logs
+ * @param lockWarningThresholdMs the time threshold to view lock held
+ * time as being "too long"
+ */
+ public InstrumentedLock(String name, Log logger, long minLoggingGapMs,
+ long lockWarningThresholdMs) {
+ this(name, logger, new ReentrantLock(),
+ minLoggingGapMs, lockWarningThresholdMs);
+ }
+
+ public InstrumentedLock(String name, Log logger, Lock lock,
+ long minLoggingGapMs, long lockWarningThresholdMs) {
+ this(name, logger, lock,
+ minLoggingGapMs, lockWarningThresholdMs, new Timer());
+ }
+
+ @VisibleForTesting
+ InstrumentedLock(String name, Log logger, Lock lock,
+ long minLoggingGapMs, long lockWarningThresholdMs, Timer clock) {
+ this.name = name;
+ this.lock = lock;
+ this.clock = clock;
+ this.logger = logger;
+ minLoggingGap = minLoggingGapMs;
+ lockWarningThreshold = lockWarningThresholdMs;
+ lastLogTimestamp = new AtomicLong(
+ clock.monotonicNow() - Math.max(minLoggingGap, lockWarningThreshold));
+ }
+
+ @Override
+ public void lock() {
+ lock.lock();
+ lockAcquireTimestamp = clock.monotonicNow();
+ }
+
+ @Override
+ public void lockInterruptibly() throws InterruptedException {
+ lock.lockInterruptibly();
+ lockAcquireTimestamp = clock.monotonicNow();
+ }
+
+ @Override
+ public boolean tryLock() {
+ if (lock.tryLock()) {
+ lockAcquireTimestamp = clock.monotonicNow();
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean tryLock(long time, TimeUnit unit) throws InterruptedException {
+ if (lock.tryLock(time, unit)) {
+ lockAcquireTimestamp = clock.monotonicNow();
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void unlock() {
+ long localLockReleaseTime = clock.monotonicNow();
+ long localLockAcquireTime = lockAcquireTimestamp;
+ lock.unlock();
+ check(localLockAcquireTime, localLockReleaseTime);
+ }
+
+ @Override
+ public Condition newCondition() {
+ return lock.newCondition();
+ }
+
+ @VisibleForTesting
+ void logWarning(long lockHeldTime, long suppressed) {
+ logger.warn(String.format("Lock held time above threshold: " +
+ "lock identifier: %s " +
+ "lockHeldTimeMs=%d ms. Suppressed %d lock warnings. " +
+ "The stack trace is: %s" ,
+ name, lockHeldTime, suppressed,
+ StringUtils.getStackTrace(Thread.currentThread())));
+ }
+
+ /**
+ * Log a warning if the lock was held for too long.
+ *
+ * Should be invoked by the caller immediately AFTER releasing the lock.
+ *
+ * @param acquireTime - timestamp just after acquiring the lock.
+ * @param releaseTime - timestamp just before releasing the lock.
+ */
+ private void check(long acquireTime, long releaseTime) {
+ if (!logger.isWarnEnabled()) {
+ return;
+ }
+
+ final long lockHeldTime = releaseTime - acquireTime;
+ if (lockWarningThreshold - lockHeldTime < 0) {
+ long now;
+ long localLastLogTs;
+ do {
+ now = clock.monotonicNow();
+ localLastLogTs = lastLogTimestamp.get();
+ long deltaSinceLastLog = now - localLastLogTs;
+ // check should print log or not
+ if (deltaSinceLastLog - minLoggingGap < 0) {
+ warningsSuppressed.incrementAndGet();
+ return;
+ }
+ } while (!lastLogTimestamp.compareAndSet(localLastLogTs, now));
+ long suppressed = warningsSuppressed.getAndSet(0);
+ logWarning(lockHeldTime, suppressed);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/011f3b24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index ffd2f8a..e5da0e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -40,6 +40,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
@@ -60,6 +61,7 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.ExtendedBlockId;
+import org.apache.hadoop.hdfs.InstrumentedLock;
import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
@@ -278,7 +280,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
this.dataStorage = storage;
this.conf = conf;
this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf);
- this.datasetLock = new AutoCloseableLock();
+ this.datasetLock = new AutoCloseableLock(
+ new InstrumentedLock(getClass().getName(), LOG,
+ conf.getTimeDuration(
+ DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY,
+ DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS),
+ 300));
// The number of volumes required for operation is the total number
// of volumes minus the number of failed volumes we can tolerate.
volFailuresTolerated = datanode.getDnConf().getVolFailuresTolerated();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/011f3b24/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3f78233..3a5de3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4273,4 +4273,12 @@
a plan.
</description>
</property>
+
+ <property>
+ <name>dfs.lock.suppress.warning.interval</name>
+ <value>10s</value>
+ <description>Instrumentation reporting long critical sections will suppress
+ consecutive warnings within this interval.</description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/011f3b24/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInstrumentedLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInstrumentedLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInstrumentedLock.java
new file mode 100644
index 0000000..f470688
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInstrumentedLock.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+
+import org.apache.hadoop.util.AutoCloseableLock;
+import org.apache.hadoop.util.Timer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import static org.mockito.Mockito.*;
+import static org.junit.Assert.*;
+
+/**
+ * A test class for InstrumentedLock.
+ */
+public class TestInstrumentedLock {
+
+ static final Log LOG = LogFactory.getLog(TestInstrumentedLock.class);
+
+ @Rule public TestName name = new TestName();
+
+ /**
+ * Test exclusive access of the lock.
+ * @throws Exception
+ */
+ @Test(timeout=10000)
+ public void testMultipleThread() throws Exception {
+ String testname = name.getMethodName();
+ InstrumentedLock lock = new InstrumentedLock(testname, LOG, 0, 300);
+ lock.lock();
+ try {
+ Thread competingThread = new Thread() {
+ @Override
+ public void run() {
+ assertFalse(lock.tryLock());
+ }
+ };
+ competingThread.start();
+ competingThread.join();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Test the correctness with try-with-resource syntax.
+ * @throws Exception
+ */
+ @Test(timeout=10000)
+ public void testTryWithResourceSyntax() throws Exception {
+ String testname = name.getMethodName();
+ final AtomicReference<Thread> lockThread = new AtomicReference<>(null);
+ Lock lock = new InstrumentedLock(testname, LOG, 0, 300) {
+ @Override
+ public void lock() {
+ super.lock();
+ lockThread.set(Thread.currentThread());
+ }
+ @Override
+ public void unlock() {
+ super.unlock();
+ lockThread.set(null);
+ }
+ };
+ AutoCloseableLock acl = new AutoCloseableLock(lock);
+ try (AutoCloseable localLock = acl.acquire()) {
+ assertEquals(acl, localLock);
+ Thread competingThread = new Thread() {
+ @Override
+ public void run() {
+ assertNotEquals(Thread.currentThread(), lockThread.get());
+ assertFalse(lock.tryLock());
+ }
+ };
+ competingThread.start();
+ competingThread.join();
+ assertEquals(Thread.currentThread(), lockThread.get());
+ }
+ assertNull(lockThread.get());
+ }
+
+ /**
+ * Test the lock logs warning when lock held time is greater than threshold
+ * and not log warning otherwise.
+ * @throws Exception
+ */
+ @Test(timeout=10000)
+ public void testLockLongHoldingReport() throws Exception {
+ String testname = name.getMethodName();
+ final AtomicLong time = new AtomicLong(0);
+ Timer mclock = new Timer() {
+ @Override
+ public long monotonicNow() {
+ return time.get();
+ }
+ };
+ Lock mlock = mock(Lock.class);
+
+ final AtomicLong wlogged = new AtomicLong(0);
+ final AtomicLong wsuppresed = new AtomicLong(0);
+ InstrumentedLock lock = new InstrumentedLock(
+ testname, LOG, mlock, 2000, 300, mclock) {
+ @Override
+ void logWarning(long lockHeldTime, long suppressed) {
+ wlogged.incrementAndGet();
+ wsuppresed.set(suppressed);
+ }
+ };
+
+ // do not log warning when the lock held time is short
+ lock.lock(); // t = 0
+ time.set(200);
+ lock.unlock(); // t = 200
+ assertEquals(0, wlogged.get());
+ assertEquals(0, wsuppresed.get());
+
+ lock.lock(); // t = 200
+ time.set(700);
+ lock.unlock(); // t = 700
+ assertEquals(1, wlogged.get());
+ assertEquals(0, wsuppresed.get());
+
+ // despite the lock held time is greater than threshold
+ // suppress the log warning due to the logging gap
+ // (not recorded in wsuppressed until next log message)
+ lock.lock(); // t = 700
+ time.set(1100);
+ lock.unlock(); // t = 1100
+ assertEquals(1, wlogged.get());
+ assertEquals(0, wsuppresed.get());
+
+ // log a warning message when the lock held time is greater the threshold
+ // and the logging time gap is satisfied. Also should display suppressed
+ // previous warnings.
+ time.set(2400);
+ lock.lock(); // t = 2400
+ time.set(2800);
+ lock.unlock(); // t = 2800
+ assertEquals(2, wlogged.get());
+ assertEquals(1, wsuppresed.get());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/43] hadoop git commit: HDFS-9847. HDFS configuration should
accept time units. Contributed by Yiqun Lin
Posted by ae...@apache.org.
HDFS-9847. HDFS configuration should accept time units. Contributed by Yiqun Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d37dc5d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d37dc5d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d37dc5d1
Branch: refs/heads/HDFS-7240
Commit: d37dc5d1b8e022a7085118a2e7066623483c293f
Parents: f388a20
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Sep 6 10:37:54 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Sep 6 10:38:04 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 47 ++++++++++++++------
.../apache/hadoop/conf/TestConfiguration.java | 46 +++++++++++++++++++
.../hadoop/hdfs/client/impl/DfsClientConf.java | 6 ++-
.../hadoop/hdfs/server/balancer/Balancer.java | 12 +++--
.../server/blockmanagement/BlockManager.java | 6 ++-
.../BlockPlacementPolicyDefault.java | 5 ++-
.../server/blockmanagement/DatanodeManager.java | 9 ++--
.../blockmanagement/DecommissionManager.java | 7 +--
.../hadoop/hdfs/server/datanode/DNConf.java | 18 ++++----
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 6 ++-
.../apache/hadoop/hdfs/server/mover/Mover.java | 12 +++--
.../hdfs/server/namenode/CheckpointConf.java | 10 +++--
.../hadoop/hdfs/server/namenode/FSImage.java | 7 +--
.../hdfs/server/namenode/ha/EditLogTailer.java | 10 +++--
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 ++-
.../src/main/resources/hdfs-default.xml | 42 ++++++++++++++---
17 files changed, 187 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 29733eb..66734b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -630,10 +630,15 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
ArrayList<String > names = new ArrayList<String>();
if (isDeprecated(name)) {
DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
- warnOnceIfDeprecated(deprecations, name);
- for (String newKey : keyInfo.newKeys) {
- if(newKey != null) {
- names.add(newKey);
+ if (keyInfo != null) {
+ if (!keyInfo.getAndSetAccessed()) {
+ logDeprecation(keyInfo.getWarningMessage(name));
+ }
+
+ for (String newKey : keyInfo.newKeys) {
+ if (newKey != null) {
+ names.add(newKey);
+ }
}
}
}
@@ -1232,11 +1237,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
}
- private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) {
- DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
- if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
- LOG_DEPRECATION.info(keyInfo.getWarningMessage(name));
- }
+ @VisibleForTesting
+ void logDeprecation(String message) {
+ LOG_DEPRECATION.info(message);
}
/**
@@ -1625,20 +1628,38 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
String vStr = get(name);
if (null == vStr) {
return defaultValue;
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
+ }
+ }
+
+ public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
+ String vStr = get(name);
+ if (null == vStr) {
+ return getTimeDurationHelper(name, defaultValue, unit);
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
}
- vStr = vStr.trim();
- return getTimeDurationHelper(name, vStr, unit);
}
private long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
+ vStr = vStr.trim();
+ vStr = StringUtils.toLowerCase(vStr);
ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
if (null == vUnit) {
- LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
+ logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + unit);
vUnit = ParsedTimeDuration.unitFor(unit);
} else {
vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
}
- return unit.convert(Long.parseLong(vStr), vUnit.unit());
+
+ long raw = Long.parseLong(vStr);
+ long converted = unit.convert(raw, vUnit.unit());
+ if (vUnit.unit().convert(converted, unit) < raw) {
+ logDeprecation("Possible loss of precision converting " + vStr
+ + vUnit.suffix() + " to " + unit + " for " + name);
+ }
+ return converted;
}
public long[] getTimeDurations(String name, TimeUnit unit) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index aa3b239..e7ebea1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -880,6 +880,15 @@ public class TestConfiguration extends TestCase {
assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS));
conf.set("test.time.X", "30");
assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS));
+ assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS));
+ assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS));
+ assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS));
+ conf.set("test.time.c", "30");
+ assertEquals(30L, conf.getTimeDuration("test.time.c", "40s", SECONDS));
+
+ // check suffix insensitive
+ conf.set("test.time.d", "30S");
+ assertEquals(30L, conf.getTimeDuration("test.time.d", 40, SECONDS));
for (Configuration.ParsedTimeDuration ptd :
Configuration.ParsedTimeDuration.values()) {
@@ -889,6 +898,43 @@ public class TestConfiguration extends TestCase {
}
}
+ public void testTimeDurationWarning() {
+ // check warn for possible loss of precision
+ final String warnFormat = "Possible loss of precision converting %s" +
+ " to %s for test.time.warn";
+ final ArrayList<String> warnchk = new ArrayList<>();
+ Configuration wconf = new Configuration(false) {
+ @Override
+ void logDeprecation(String message) {
+ warnchk.add(message);
+ }
+ };
+ String[] convDAYS = new String[]{"23h", "30m", "40s", "10us", "40000ms"};
+ for (String s : convDAYS) {
+ wconf.set("test.time.warn", s);
+ assertEquals(0, wconf.getTimeDuration("test.time.warn", 1, DAYS));
+ }
+ for (int i = 0; i < convDAYS.length; ++i) {
+ String wchk = String.format(warnFormat, convDAYS[i], "DAYS");
+ assertEquals(wchk, warnchk.get(i));
+ }
+
+ warnchk.clear();
+ wconf.setTimeDuration("test.time.warn", 1441, MINUTES);
+ assertEquals(1, wconf.getTimeDuration("test.time.warn", 0, DAYS));
+ assertEquals(24, wconf.getTimeDuration("test.time.warn", 0, HOURS));
+ String dchk = String.format(warnFormat, "1441m", "DAYS");
+ assertEquals(dchk, warnchk.get(0));
+ String hchk = String.format(warnFormat, "1441m", "HOURS");
+ assertEquals(hchk, warnchk.get(1));
+ assertEquals(1441, wconf.getTimeDuration("test.time.warn", 0, MINUTES));
+ // no warning
+ assertEquals(2, warnchk.size());
+ assertEquals(86460, wconf.getTimeDuration("test.time.warn", 0, SECONDS));
+ // no warning
+ assertEquals(2, warnchk.size());
+ }
+
public void testPattern() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index ded989c..4f4c7b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -85,6 +85,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* DFSClient configuration.
@@ -233,9 +234,10 @@ public class DfsClientConf {
connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
- datanodeRestartTimeout = conf.getLong(
+ datanodeRestartTimeout = conf.getTimeDuration(
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
- DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000;
+ DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
slowIoWarningThresholdMs = conf.getLong(
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 8d94ed5..2037d01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -662,10 +663,13 @@ public class Balancer {
static int run(Collection<URI> namenodes, final BalancerParameters p,
Configuration conf) throws IOException, InterruptedException {
final long sleeptime =
- conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
- conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
+ conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 2000 +
+ conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
LOG.info("namenodes = " + namenodes);
LOG.info("parameters = " + p);
LOG.info("included nodes = " + p.getIncludedNodes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 9e3b935..1362c0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -42,6 +42,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import javax.management.ObjectName;
@@ -405,8 +406,9 @@ public class BlockManager implements BlockStatsMXBean {
this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
this.replicationRecheckInterval =
- conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000L;
+ conf.getTimeDuration(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000L;
this.storageInfoDefragmentInterval =
conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 19f4dde..abfa782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.apache.hadoop.util.Time.monotonicNow;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -89,9 +90,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
this.stats = stats;
this.clusterMap = clusterMap;
this.host2datanodeMap = host2datanodeMap;
- this.heartbeatInterval = conf.getLong(
+ this.heartbeatInterval = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000;
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000;
this.tolerateHeartbeatMultiplier = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY,
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index fffe29c..2d6547f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -58,6 +58,7 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
/**
* Manage datanodes, include decommission and other activities.
@@ -234,9 +235,9 @@ public class DatanodeManager {
dnsToSwitchMapping.resolve(locations);
}
- heartbeatIntervalSeconds = conf.getLong(
+ heartbeatIntervalSeconds = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
heartbeatRecheckInterval = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
@@ -289,9 +290,9 @@ public class DatanodeManager {
" = '" + staleInterval + "' is invalid. " +
"It should be a positive non-zero value.");
- final long heartbeatIntervalSeconds = conf.getLong(
+ final long heartbeatIntervalSeconds = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
// The stale interval value cannot be smaller than
// 3 times of heartbeat interval
final long minStaleInterval = conf.getInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index c456aba..6436fab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -134,9 +134,10 @@ public class DecommissionManager {
* @param conf
*/
void activate(Configuration conf) {
- final int intervalSecs =
- conf.getInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT);
+ final int intervalSecs = (int) conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS);
checkArgument(intervalSecs >= 0, "Cannot set a negative " +
"value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 11197e6..757d7f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
import org.apache.hadoop.security.SaslPropertiesResolver;
+import java.util.concurrent.TimeUnit;
+
/**
* Simple class encapsulating all of the configuration that the DataNode
* loads at startup time.
@@ -184,9 +186,9 @@ public class DNConf {
DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY,
DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
- long initBRDelay = conf.getLong(
+ long initBRDelay = conf.getTimeDuration(
DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
- DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT) * 1000L;
+ DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS) * 1000L;
if (initBRDelay >= blockReportInterval) {
initBRDelay = 0;
DataNode.LOG.info("dfs.blockreport.initialDelay is "
@@ -195,12 +197,12 @@ public class DNConf {
}
initialBlockReportDelayMs = initBRDelay;
- heartBeatInterval = conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000L;
+ heartBeatInterval = conf.getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000L;
long confLifelineIntervalMs =
conf.getLong(DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY,
- 3 * conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT)) * 1000L;
+ 3 * conf.getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS)) * 1000L;
if (confLifelineIntervalMs <= heartBeatInterval) {
confLifelineIntervalMs = 3 * heartBeatInterval;
DataNode.LOG.warn(
@@ -245,9 +247,9 @@ public class DNConf {
DFS_DATANODE_NON_LOCAL_LAZY_PERSIST,
DFS_DATANODE_NON_LOCAL_LAZY_PERSIST_DEFAULT);
- this.bpReadyTimeout = conf.getLong(
+ this.bpReadyTimeout = conf.getTimeDuration(
DFS_DATANODE_BP_READY_TIMEOUT_KEY,
- DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT);
+ DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT, TimeUnit.SECONDS);
this.volFailuresTolerated =
conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 08a1fc0..0025040 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1024,8 +1024,8 @@ public class DataNode extends ReconfigurableBase
return;
}
String reason = null;
- if (conf.getInt(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
- DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT) < 0) {
+ if (conf.getTimeDuration(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
+ DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT, TimeUnit.SECONDS) < 0) {
reason = "verification is turned off by configuration";
} else if ("SimulatedFSDataset".equals(data.getClass().getSimpleName())) {
reason = "verifcation is not supported by SimulatedFSDataset";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 7c8985e..f9ebab9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -414,8 +414,10 @@ public class DirectoryScanner implements Runnable {
DirectoryScanner(DataNode datanode, FsDatasetSpi<?> dataset, Configuration conf) {
this.datanode = datanode;
this.dataset = dataset;
- int interval = conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
- DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT);
+ int interval = (int) conf.getTimeDuration(
+ DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
+ DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS);
scanPeriodMsecs = interval * MILLIS_PER_SECOND; //msec
int throttle =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index fdb6cfa..0c85374 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -64,6 +64,7 @@ import java.net.InetSocketAddress;
import java.net.URI;
import java.text.DateFormat;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@InterfaceAudience.Private
@@ -602,10 +603,13 @@ public class Mover {
static int run(Map<URI, List<Path>> namenodes, Configuration conf)
throws IOException, InterruptedException {
final long sleeptime =
- conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
- conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
+ conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 2000 +
+ conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
AtomicInteger retryCount = new AtomicInteger(0);
LOG.info("namenodes = " + namenodes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
index c30730b..aeee87d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import com.google.common.collect.ImmutableList;
+import java.util.concurrent.TimeUnit;
+
@InterfaceAudience.Private
public class CheckpointConf {
private static final Log LOG = LogFactory.getLog(CheckpointConf.class);
@@ -52,12 +54,12 @@ public class CheckpointConf {
private double quietMultiplier;
public CheckpointConf(Configuration conf) {
- checkpointCheckPeriod = conf.getLong(
+ checkpointCheckPeriod = conf.getTimeDuration(
DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY,
- DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT);
+ DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT, TimeUnit.SECONDS);
- checkpointPeriod = conf.getLong(DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ checkpointPeriod = conf.getTimeDuration(DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
+ DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT, TimeUnit.SECONDS);
checkpointTxnCount = conf.getLong(DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
maxRetriesOnMergeError = conf.getInt(DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index b637105..d02a6ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -34,6 +34,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -798,9 +799,9 @@ public class FSImage implements Closeable {
*/
private boolean needsResaveBasedOnStaleCheckpoint(
File imageFile, long numEditsLoaded) {
- final long checkpointPeriod = conf.getLong(
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ final long checkpointPeriod = conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT, TimeUnit.SECONDS);
final long checkpointTxnCount = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 95c3d58..f57cb4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -146,8 +146,9 @@ public class EditLogTailer {
lastLoadTimeMs = monotonicNow();
- logRollPeriodMs = conf.getInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT) * 1000;
+ logRollPeriodMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
+ DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
List<RemoteNameNodeInfo> nns = Collections.emptyList();
if (logRollPeriodMs >= 0) {
try {
@@ -172,8 +173,9 @@ public class EditLogTailer {
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY + " is negative.");
}
- sleepTimeMs = conf.getInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT) * 1000;
+ sleepTimeMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
+ DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
rollEditsTimeoutMs = conf.getInt(
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index ba09e8b..e6e0fbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -34,6 +34,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
@@ -742,9 +743,10 @@ public class DFSAdmin extends FsShell {
long timeWindow = 0;
long txGap = 0;
if (argv.length > 1 && "-beforeShutdown".equals(argv[1])) {
- final long checkpointPeriod = dfsConf.getLong(
+ final long checkpointPeriod = dfsConf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT,
+ TimeUnit.SECONDS);
final long checkpointTxnCount = dfsConf.getLong(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0c2c743..4eca80c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -637,8 +637,12 @@
</property>
<property>
- <name>dfs.blockreport.initialDelay</name> <value>0</value>
- <description>Delay for first block report in seconds.</description>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>0</value>
+ <description>
+ Delay for first block report in seconds. Support multiple time unit
+ suffix(case insensitive), as described in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -681,6 +685,8 @@
<value>21600</value>
<description>Interval in seconds for Datanode to scan data directories and
reconcile the difference between blocks in memory and on the disk.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -715,7 +721,13 @@
<property>
<name>dfs.heartbeat.interval</name>
<value>3</value>
- <description>Determines datanode heartbeat interval in seconds.</description>
+ <description>
+ Determines datanode heartbeat interval in seconds.
+ Can use the following suffix (case insensitive):
+ ms(millis), s(sec), m(min), h(hour), d(day)
+ to specify the time (such as 2s, 2m, 1h, etc.).
+ Or provide complete number in seconds (such as 30 for 30 seconds).
+ </description>
</property>
<property>
@@ -920,7 +932,9 @@
<name>dfs.namenode.decommission.interval</name>
<value>30</value>
<description>Namenode periodicity in seconds to check if decommission is
- complete.</description>
+ complete. Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -949,7 +963,9 @@
<name>dfs.namenode.replication.interval</name>
<value>3</value>
<description>The periodicity in seconds with which the namenode computes
- replication work for datanodes. </description>
+ replication work for datanodes. Support multiple time unit suffix(case insensitive),
+ as described in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -1044,7 +1060,10 @@
<property>
<name>dfs.namenode.checkpoint.period</name>
<value>3600</value>
- <description>The number of seconds between two periodic checkpoints.
+ <description>
+ The number of seconds between two periodic checkpoints.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1062,7 +1081,8 @@
<value>60</value>
<description>The SecondaryNameNode and CheckpointNode will poll the NameNode
every 'dfs.namenode.checkpoint.check.period' seconds to query the number
- of uncheckpointed transactions.
+ of uncheckpointed transactions. Support multiple time unit suffix(case insensitive),
+ as described in dfs.heartbeat.interval.
</description>
</property>
@@ -1408,6 +1428,8 @@
the datanode dead and invoking the normal recovery mechanisms.
The notification is sent by a datanode when it is being shutdown
using the shutdownDatanode admin command with the upgrade option.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1475,6 +1497,8 @@
log segments, the StandbyNode will only be as up-to-date as how
often the logs are rolled. Note that failover triggers a log roll
so the StandbyNode will be up to date before it becomes active.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1484,6 +1508,8 @@
<description>
How often, in seconds, the StandbyNode should check for new
finalized log segments in the shared edits log.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -2918,6 +2944,8 @@
received request. Setting this to 0 fails requests right away if the
datanode is not yet registered with the namenode. This wait time
reduces initial request failures after datanode restart.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/43] hadoop git commit: HADOOP-13558. UserGroupInformation created
from a Subject incorrectly tries to renew the Kerberos ticket. Contributed by
Xiao Chen.
Posted by ae...@apache.org.
HADOOP-13558. UserGroupInformation created from a Subject incorrectly tries to renew the Kerberos ticket. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/680be58a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/680be58a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/680be58a
Branch: refs/heads/HDFS-7240
Commit: 680be58aac03a9ffab6b07c8fde9602ddb9dc858
Parents: 5f23abf
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 20:25:26 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 20:54:17 2016 -0700
----------------------------------------------------------------------
.../hadoop/security/UserGroupInformation.java | 22 +++++++++++++++---
.../security/TestUserGroupInformation.java | 24 ++++++++++++++++++++
2 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/680be58a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index fe6fbe4..ed3a9d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -632,9 +632,24 @@ public class UserGroupInformation {
* @param subject the user's subject
*/
UserGroupInformation(Subject subject) {
+ this(subject, false);
+ }
+
+ /**
+ * Create a UGI from the given subject.
+ * @param subject the subject
+ * @param externalKeyTab if the subject's keytab is managed by the user.
+ * Setting this to true will prevent UGI from attempting
+ * to login the keytab, or to renew it.
+ */
+ private UserGroupInformation(Subject subject, final boolean externalKeyTab) {
this.subject = subject;
this.user = subject.getPrincipals(User.class).iterator().next();
- this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject);
+ if (externalKeyTab) {
+ this.isKeytab = false;
+ } else {
+ this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject);
+ }
this.isKrbTkt = KerberosUtil.hasKerberosTicket(subject);
}
@@ -850,10 +865,11 @@ public class UserGroupInformation {
newLoginContext(authenticationMethod.getLoginAppName(),
subject, new HadoopConfiguration());
login.login();
- UserGroupInformation realUser = new UserGroupInformation(subject);
+ LOG.debug("Assuming keytab is managed externally since logged in from"
+ + " subject.");
+ UserGroupInformation realUser = new UserGroupInformation(subject, true);
realUser.setLogin(login);
realUser.setAuthenticationMethod(authenticationMethod);
- realUser = new UserGroupInformation(login.getSubject());
// If the HADOOP_PROXY_USER environment variable or property
// is specified, create a proxy user as the logged in user.
String proxyUser = System.getenv(HADOOP_PROXY_USER);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/680be58a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index a306d35..e45d70d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -37,6 +37,7 @@ import org.junit.Test;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.kerberos.KeyTab;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
@@ -1030,4 +1031,27 @@ public class TestUserGroupInformation {
assertTrue(credsugiTokens.contains(token1));
assertTrue(credsugiTokens.contains(token2));
}
+
+ @Test
+ public void testCheckTGTAfterLoginFromSubject() throws Exception {
+ // security on, default is remove default realm
+ SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ UserGroupInformation.setConfiguration(conf);
+
+ // Login from a pre-set subject with a keytab
+ final Subject subject = new Subject();
+ KeyTab keytab = KeyTab.getInstance();
+ subject.getPrivateCredentials().add(keytab);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException {
+ UserGroupInformation.loginUserFromSubject(subject);
+ // this should not throw.
+ UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+ return null;
+ }
+ });
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/43] hadoop git commit: HDFS-10831. Add log when
URLConnectionFactory.openConnection failed. Contributed by yunjiong zhao.
Posted by ae...@apache.org.
HDFS-10831. Add log when URLConnectionFactory.openConnection failed. Contributed by yunjiong zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b07c266d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b07c266d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b07c266d
Branch: refs/heads/HDFS-7240
Commit: b07c266dca7f303c793b432a0738d593728cf2b3
Parents: 011f3b2
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Sep 8 18:30:18 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Sep 8 18:30:18 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b07c266d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
index 975f72e..96095db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
@@ -183,6 +183,7 @@ public class URLConnectionFactory {
return openConnection(url, false);
} catch (AuthenticationException e) {
// Unreachable
+ LOG.error("Open connection {} failed", url, e);
return null;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/43] hadoop git commit: YARN-5576. Allow resource localization
while container is running. Contributed by Jian He.
Posted by ae...@apache.org.
YARN-5576. Allow resource localization while container is running. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6fcfe28
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6fcfe28
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6fcfe28
Branch: refs/heads/HDFS-7240
Commit: e6fcfe28e304062c7d09231db757acb2953703ce
Parents: 62a9667
Author: Varun Vasudev <vv...@apache.org>
Authored: Tue Sep 6 20:01:45 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Tue Sep 6 20:01:45 2016 +0530
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 9 +
.../hadoop/yarn/server/nodemanager/Context.java | 2 +
.../nodemanager/DefaultContainerExecutor.java | 6 +
.../nodemanager/DockerContainerExecutor.java | 6 +
.../nodemanager/LinuxContainerExecutor.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 12 ++
.../containermanager/ContainerManagerImpl.java | 70 +++++--
.../containermanager/container/Container.java | 8 +-
.../container/ContainerImpl.java | 207 ++++++++-----------
.../launcher/ContainerLaunch.java | 4 +-
.../localizer/ResourceLocalizationService.java | 21 +-
.../containermanager/localizer/ResourceSet.java | 192 +++++++++++++++++
.../ContainerLocalizationRequestEvent.java | 2 +-
.../localizer/event/LocalizationEventType.java | 2 +-
.../timelineservice/NMTimelinePublisher.java | 2 +-
.../nodemanager/DummyContainerManager.java | 2 +-
.../TestContainerManagerWithLCE.java | 13 +-
.../amrmproxy/BaseAMRMProxyTest.java | 6 +
.../containermanager/TestContainerManager.java | 137 +++++++++++-
.../TestResourceLocalizationService.java | 7 +-
.../TestContainersMonitorResourceChange.java | 7 +
.../nodemanager/webapp/MockContainer.java | 17 +-
22 files changed, 577 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 818b0ea..918c30a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -182,6 +182,15 @@ public abstract class ContainerExecutor implements Configurable {
throws IOException, InterruptedException;
/**
+ * Create a symlink file which points to the target.
+ * @param target The target for symlink
+ * @param symlink the symlink file
+ * @throws IOException Error when creating symlinks
+ */
+ public abstract void symLink(String target, String symlink)
+ throws IOException;
+
+ /**
* Check if a container is alive.
* @param ctx Encapsulates information necessary for container liveness check.
* @return true if container is still alive
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 88bc29c..e888393 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -126,4 +126,6 @@ public interface Context {
void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher);
NMTimelinePublisher getNMTimelinePublisher();
+
+ ContainerExecutor getContainerExecutor();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 13ad9ac..9a0549d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -511,6 +512,11 @@ public class DefaultContainerExecutor extends ContainerExecutor {
}
}
+ @Override
+ public void symLink(String target, String symlink) throws IOException {
+ FileUtil.symLink(target, symlink);
+ }
+
/** Permissions for user dir.
* $local.dir/usercache/$user */
static final short USER_PERM = (short)0750;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index ebf9566..1390214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -490,6 +490,12 @@ public class DockerContainerExecutor extends ContainerExecutor {
}
}
+ @Override
+ public void symLink(String target, String symlink)
+ throws IOException {
+
+ }
+
/**
* Converts a directory list to a docker mount string
* @param dirs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 6890b25..cc12b20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -683,6 +683,11 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ public void symLink(String target, String symlink) {
+
+ }
+
+ @Override
public boolean isContainerAlive(ContainerLivenessContext ctx)
throws IOException {
String user = ctx.getUser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 280a086..7f13334 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -343,6 +343,9 @@ public class NodeManager extends CompositeService
this.context = createNMContext(containerTokenSecretManager,
nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
+
+ ((NMContext)context).setContainerExecutor(exec);
+
nodeLabelsProvider = createNodeLabelsProvider(conf);
if (null == nodeLabelsProvider) {
@@ -509,6 +512,7 @@ public class NodeManager extends CompositeService
private OpportunisticContainerAllocator containerAllocator;
private final QueuingContext queuingContext;
+ private ContainerExecutor executor;
private NMTimelinePublisher nmTimelinePublisher;
@@ -702,6 +706,14 @@ public class NodeManager extends CompositeService
public NMTimelinePublisher getNMTimelinePublisher() {
return nmTimelinePublisher;
}
+
+ public ContainerExecutor getContainerExecutor() {
+ return this.executor;
+ }
+
+ public void setContainerExecutor(ContainerExecutor executor) {
+ this.executor = executor;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8c060bc..52d8566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -18,25 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager;
-import static org.apache.hadoop.service.Service.STATE.STARTED;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -76,6 +59,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -129,7 +113,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.SignalContainersLauncherEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType;
@@ -154,8 +140,25 @@ import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ByteString;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import static org.apache.hadoop.service.Service.STATE.STARTED;
public class ContainerManagerImpl extends CompositeService implements
ContainerManager {
@@ -1525,6 +1528,31 @@ public class ContainerManagerImpl extends CompositeService implements
public ResourceLocalizationResponse localize(
ResourceLocalizationRequest request) throws YarnException, IOException {
+ ContainerId containerId = request.getContainerId();
+ Container container = context.getContainers().get(containerId);
+ if (container == null) {
+ throw new YarnException("Specified " + containerId + " does not exist!");
+ }
+ if (!container.getContainerState()
+ .equals(org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING)) {
+ throw new YarnException(
+ containerId + " is at " + container.getContainerState()
+ + " state. Not able to localize new resources.");
+ }
+
+ try {
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ container.getResourceSet().addResources(request.getLocalResources());
+ if (req != null && !req.isEmpty()) {
+ dispatcher.getEventHandler()
+ .handle(new ContainerLocalizationRequestEvent(container, req));
+ }
+ } catch (URISyntaxException e) {
+ LOG.info("Error when parsing local resource URI for " + containerId, e);
+ throw new YarnException(e);
+ }
+
return ResourceLocalizationResponse.newInstance();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 84d3cb2..c4cea18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
-import java.util.List;
-import java.util.Map;
-
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -31,6 +28,10 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
+
+import java.util.List;
+import java.util.Map;
public interface Container extends EventHandler<ContainerEvent> {
@@ -74,4 +75,5 @@ public interface Container extends EventHandler<ContainerEvent> {
Priority getPriority();
+ ResourceSet getResourceSet();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 0244d90..ce9e581 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -18,18 +18,15 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -65,6 +62,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
@@ -124,20 +122,7 @@ public class ContainerImpl implements Container {
private final Configuration daemonConf;
private static final Log LOG = LogFactory.getLog(ContainerImpl.class);
- private final Map<LocalResourceRequest,List<String>> pendingResources =
- new HashMap<LocalResourceRequest,List<String>>();
- private final Map<Path,List<String>> localizedResources =
- new HashMap<Path,List<String>>();
- private final List<LocalResourceRequest> publicRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final List<LocalResourceRequest> privateRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final List<LocalResourceRequest> appRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final Map<LocalResourceRequest, Path> resourcesToBeUploaded =
- new ConcurrentHashMap<LocalResourceRequest, Path>();
- private final Map<LocalResourceRequest, Boolean> resourcesUploadPolicies =
- new ConcurrentHashMap<LocalResourceRequest, Boolean>();
+
// whether container has been recovered after a restart
private RecoveredContainerStatus recoveredStatus =
@@ -145,6 +130,7 @@ public class ContainerImpl implements Container {
// whether container was marked as killed after recovery
private boolean recoveredAsKilled = false;
private Context context;
+ private ResourceSet resourceSet;
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
ContainerLaunchContext launchContext, Credentials creds,
@@ -204,6 +190,7 @@ public class ContainerImpl implements Container {
stateMachine = stateMachineFactory.make(this);
this.context = context;
+ this.resourceSet = new ResourceSet();
}
// constructor for a recovered container
@@ -313,6 +300,12 @@ public class ContainerImpl implements Container {
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
new RetryFailureTransition())
.addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ new ResourceLocalizedWhileRunningTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
+ ContainerEventType.RESOURCE_FAILED,
+ new ResourceLocalizationFailedWhileRunningTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.RUNNING, ContainerState.KILLING,
@@ -470,7 +463,7 @@ public class ContainerImpl implements Container {
try {
if (ContainerState.LOCALIZED == getContainerState()
|| ContainerState.RELAUNCHING == getContainerState()) {
- return localizedResources;
+ return resourceSet.getLocalizedResources();
} else {
return null;
}
@@ -591,6 +584,11 @@ public class ContainerImpl implements Container {
this.logDir = logDir;
}
+ @Override
+ public ResourceSet getResourceSet() {
+ return this.resourceSet;
+ }
+
@SuppressWarnings("unchecked")
private void sendFinishedEvents() {
// Inform the application
@@ -653,7 +651,7 @@ public class ContainerImpl implements Container {
for (String s : diags) {
this.diagnostics.append(s);
}
- if (isRetryContextSet() && diagnostics.length() > diagnosticsMaxSize) {
+ if (diagnostics.length() > diagnosticsMaxSize) {
diagnostics.delete(0, diagnostics.length() - diagnosticsMaxSize);
}
try {
@@ -667,17 +665,7 @@ public class ContainerImpl implements Container {
@SuppressWarnings("unchecked") // dispatcher not typed
public void cleanup() {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
- new HashMap<LocalResourceVisibility,
- Collection<LocalResourceRequest>>();
- if (!publicRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
- }
- if (!privateRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
- }
- if (!appRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
- }
+ resourceSet.getAllResourcesByVisibility();
dispatcher.getEventHandler().handle(
new ContainerLocalizationCleanupEvent(this, rsrc));
}
@@ -697,7 +685,7 @@ public class ContainerImpl implements Container {
* message.
*
* If there are resources to localize, sends a
- * ContainerLocalizationRequest (INIT_CONTAINER_RESOURCES)
+ * ContainerLocalizationRequest (LOCALIZE_CONTAINER_RESOURCES)
* to the ResourceLocalizationManager and enters LOCALIZING state.
*
* If there are no resources to localize, sends LAUNCH_CONTAINER event
@@ -749,39 +737,15 @@ public class ContainerImpl implements Container {
}
container.containerLocalizationStartTime = clock.getTime();
+
// Send requests for public, private resources
Map<String,LocalResource> cntrRsrc = ctxt.getLocalResources();
if (!cntrRsrc.isEmpty()) {
try {
- for (Map.Entry<String,LocalResource> rsrc : cntrRsrc.entrySet()) {
- try {
- LocalResourceRequest req =
- new LocalResourceRequest(rsrc.getValue());
- List<String> links = container.pendingResources.get(req);
- if (links == null) {
- links = new ArrayList<String>();
- container.pendingResources.put(req, links);
- }
- links.add(rsrc.getKey());
- storeSharedCacheUploadPolicy(container, req, rsrc.getValue()
- .getShouldBeUploadedToSharedCache());
- switch (rsrc.getValue().getVisibility()) {
- case PUBLIC:
- container.publicRsrcs.add(req);
- break;
- case PRIVATE:
- container.privateRsrcs.add(req);
- break;
- case APPLICATION:
- container.appRsrcs.add(req);
- break;
- }
- } catch (URISyntaxException e) {
- LOG.info("Got exception parsing " + rsrc.getKey()
- + " and value " + rsrc.getValue());
- throw e;
- }
- }
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ container.resourceSet.addResources(ctxt.getLocalResources());
+ container.dispatcher.getEventHandler().handle(
+ new ContainerLocalizationRequestEvent(container, req));
} catch (URISyntaxException e) {
// malformed resource; abort container launch
LOG.warn("Failed to parse resource-request", e);
@@ -789,21 +753,6 @@ public class ContainerImpl implements Container {
container.metrics.endInitingContainer();
return ContainerState.LOCALIZATION_FAILED;
}
- Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
- new LinkedHashMap<LocalResourceVisibility,
- Collection<LocalResourceRequest>>();
- if (!container.publicRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.PUBLIC, container.publicRsrcs);
- }
- if (!container.privateRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.PRIVATE, container.privateRsrcs);
- }
- if (!container.appRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.APPLICATION, container.appRsrcs);
- }
-
- container.dispatcher.getEventHandler().handle(
- new ContainerLocalizationRequestEvent(container, req));
return ContainerState.LOCALIZING;
} else {
container.sendLaunchEvent();
@@ -814,27 +763,6 @@ public class ContainerImpl implements Container {
}
/**
- * Store the resource's shared cache upload policies
- * Given LocalResourceRequest can be shared across containers in
- * LocalResourcesTrackerImpl, we preserve the upload policies here.
- * In addition, it is possible for the application to create several
- * "identical" LocalResources as part of
- * ContainerLaunchContext.setLocalResources with different symlinks.
- * There is a corner case where these "identical" local resources have
- * different upload policies. For that scenario, upload policy will be set to
- * true as long as there is at least one LocalResource entry with
- * upload policy set to true.
- */
- private static void storeSharedCacheUploadPolicy(ContainerImpl container,
- LocalResourceRequest resourceRequest, Boolean uploadPolicy) {
- Boolean storedUploadPolicy =
- container.resourcesUploadPolicies.get(resourceRequest);
- if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) {
- container.resourcesUploadPolicies.put(resourceRequest, uploadPolicy);
- }
- }
-
- /**
* Transition when one of the requested resources for this container
* has been successfully localized.
*/
@@ -847,22 +775,21 @@ public class ContainerImpl implements Container {
ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event;
LocalResourceRequest resourceRequest = rsrcEvent.getResource();
Path location = rsrcEvent.getLocation();
- List<String> syms = container.pendingResources.remove(resourceRequest);
+ List<String> syms =
+ container.resourceSet.resourceLocalized(resourceRequest, location);
if (null == syms) {
- LOG.warn("Localized unknown resource " + resourceRequest +
- " for container " + container.containerId);
- assert false;
- // fail container?
+ LOG.info("Localized resource " + resourceRequest +
+ " for container " + container.containerId);
return ContainerState.LOCALIZING;
}
- container.localizedResources.put(location, syms);
// check to see if this resource should be uploaded to the shared cache
// as well
if (shouldBeUploadedToSharedCache(container, resourceRequest)) {
- container.resourcesToBeUploaded.put(resourceRequest, location);
+ container.resourceSet.getResourcesToBeUploaded()
+ .put(resourceRequest, location);
}
- if (!container.pendingResources.isEmpty()) {
+ if (!container.resourceSet.getPendingResources().isEmpty()) {
return ContainerState.LOCALIZING;
}
@@ -884,7 +811,8 @@ public class ContainerImpl implements Container {
&& container.recoveredStatus != RecoveredContainerStatus.COMPLETED) {
// kick off uploads to the shared cache
container.dispatcher.getEventHandler().handle(
- new SharedCacheUploadEvent(container.resourcesToBeUploaded, container
+ new SharedCacheUploadEvent(
+ container.resourceSet.getResourcesToBeUploaded(), container
.getLaunchContext(), container.getUser(),
SharedCacheUploadEventType.UPLOAD));
}
@@ -894,6 +822,56 @@ public class ContainerImpl implements Container {
}
/**
+ * Resource is localized while the container is running - create symlinks
+ */
+ static class ResourceLocalizedWhileRunningTransition
+ extends ContainerTransition {
+
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ ContainerResourceLocalizedEvent rsrcEvent =
+ (ContainerResourceLocalizedEvent) event;
+ List<String> links = container.resourceSet
+ .resourceLocalized(rsrcEvent.getResource(), rsrcEvent.getLocation());
+ // creating symlinks.
+ for (String link : links) {
+ try {
+ String linkFile = new Path(container.workDir, link).toString();
+ if (new File(linkFile).exists()) {
+ LOG.info("Symlink file already exists: " + linkFile);
+ } else {
+ container.context.getContainerExecutor()
+ .symLink(rsrcEvent.getLocation().toString(), linkFile);
+ LOG.info("Created symlink: " + linkFile + " -> " + rsrcEvent
+ .getLocation());
+ }
+ } catch (IOException e) {
+ String message = String
+ .format("Error when creating symlink %s -> %s", link,
+ rsrcEvent.getLocation());
+ LOG.error(message, e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Resource localization failed while the container is running.
+ */
+ static class ResourceLocalizationFailedWhileRunningTransition
+ extends ContainerTransition {
+
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ ContainerResourceFailedEvent failedEvent =
+ (ContainerResourceFailedEvent) event;
+ container.resourceSet
+ .resourceLocalizationFailed(failedEvent.getResource());
+ container.addDiagnostics(failedEvent.getDiagnosticMessage());
+ }
+ }
+
+ /**
* Transition from LOCALIZED state to RUNNING state upon receiving
* a CONTAINER_LAUNCHED event
*/
@@ -1136,17 +1114,10 @@ public class ContainerImpl implements Container {
SingleArcTransition<ContainerImpl, ContainerEvent> {
@Override
public void transition(ContainerImpl container, ContainerEvent event) {
- ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event;
- List<String> syms =
- container.pendingResources.remove(rsrcEvent.getResource());
- if (null == syms) {
- LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.containerId);
- assert false;
- // fail container?
- return;
- }
- container.localizedResources.put(rsrcEvent.getLocation(), syms);
+ ContainerResourceLocalizedEvent rsrcEvent =
+ (ContainerResourceLocalizedEvent) event;
+ container.resourceSet
+ .resourceLocalized(rsrcEvent.getResource(), rsrcEvent.getLocation());
}
}
@@ -1402,7 +1373,7 @@ public class ContainerImpl implements Container {
*/
private static boolean shouldBeUploadedToSharedCache(ContainerImpl container,
LocalResourceRequest resource) {
- return container.resourcesUploadPolicies.get(resource);
+ return container.resourceSet.getResourcesUploadPolicies().get(resource);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 14190fc..d8239ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1201,16 +1201,16 @@ public class ContainerLaunch implements Callable<Integer> {
private void recordContainerLogDir(ContainerId containerId,
String logDir) throws IOException{
+ container.setLogDir(logDir);
if (container.isRetryContextSet()) {
- container.setLogDir(logDir);
context.getNMStateStore().storeContainerLogDir(containerId, logDir);
}
}
private void recordContainerWorkDir(ContainerId containerId,
String workDir) throws IOException{
+ container.setWorkDir(workDir);
if (container.isRetryContextSet()) {
- container.setWorkDir(workDir);
context.getNMStateStore().storeContainerWorkDir(containerId, workDir);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 0ec2f29..b281ef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -112,6 +112,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationEvent;
@@ -135,7 +136,6 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.Re
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredUserResources;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerBuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload;
import com.google.common.annotations.VisibleForTesting;
@@ -419,7 +419,7 @@ public class ResourceLocalizationService extends CompositeService
handleInitApplicationResources(
((ApplicationLocalizationEvent)event).getApplication());
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
handleInitContainerResources((ContainerLocalizationRequestEvent) event);
break;
case CONTAINER_RESOURCES_LOCALIZED:
@@ -469,6 +469,13 @@ public class ResourceLocalizationService extends CompositeService
private void handleInitContainerResources(
ContainerLocalizationRequestEvent rsrcReqs) {
Container c = rsrcReqs.getContainer();
+ EnumSet<ContainerState> set =
+ EnumSet.of(ContainerState.LOCALIZING, ContainerState.RUNNING);
+ if (!set.contains(c.getContainerState())) {
+ LOG.warn(c.getContainerId() + " is at " + c.getContainerState()
+ + " state, do not localize resources.");
+ return;
+ }
// create a loading cache for the file statuses
LoadingCache<Path,Future<FileStatus>> statCache =
CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
@@ -538,7 +545,7 @@ public class ResourceLocalizationService extends CompositeService
}
String locId = c.getContainerId().toString();
localizerTracker.cleanupPrivLocalizers(locId);
-
+
// Delete the container directories
String userName = c.getUser();
String containerIDStr = c.toString();
@@ -747,6 +754,14 @@ public class ResourceLocalizationService extends CompositeService
case APPLICATION:
synchronized (privLocalizers) {
LocalizerRunner localizer = privLocalizers.get(locId);
+ if (localizer != null && localizer.killContainerLocalizer.get()) {
+ // Old localizer thread has been stopped, remove it and creates
+ // a new localizer thread.
+ LOG.info("New " + event.getType() + " localize request for "
+ + locId + ", remove old private localizer.");
+ cleanupPrivLocalizers(locId);
+ localizer = null;
+ }
if (null == localizer) {
LOG.info("Created localizer for " + locId);
localizer = new LocalizerRunner(req.getContext(), locId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
new file mode 100644
index 0000000..a41ee20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * All Resources requested by the container.
+ */
+public class ResourceSet {
+
+ private static final Log LOG = LogFactory.getLog(ResourceSet.class);
+
+ // resources by localization state (localized, pending, failed)
+ private Map<Path, List<String>> localizedResources =
+ new ConcurrentHashMap<>();
+ private Map<LocalResourceRequest, List<String>> pendingResources =
+ new ConcurrentHashMap<>();
+ private Set<LocalResourceRequest> resourcesFailedToBeLocalized =
+ new HashSet<>();
+
+ // resources by visibility (public, private, app)
+ private final List<LocalResourceRequest> publicRsrcs =
+ new ArrayList<>();
+ private final List<LocalResourceRequest> privateRsrcs =
+ new ArrayList<>();
+ private final List<LocalResourceRequest> appRsrcs =
+ new ArrayList<>();
+
+ private final Map<LocalResourceRequest, Path> resourcesToBeUploaded =
+ new ConcurrentHashMap<>();
+ private final Map<LocalResourceRequest, Boolean> resourcesUploadPolicies =
+ new ConcurrentHashMap<>();
+
+ public Map<LocalResourceVisibility, Collection<LocalResourceRequest>>
+ addResources(Map<String, LocalResource> localResourceMap)
+ throws URISyntaxException {
+ if (localResourceMap == null || localResourceMap.isEmpty()) {
+ return null;
+ }
+ Map<LocalResourceRequest, List<String>> allResources = new HashMap<>();
+ List<LocalResourceRequest> publicList = new ArrayList<>();
+ List<LocalResourceRequest> privateList = new ArrayList<>();
+ List<LocalResourceRequest> appList = new ArrayList<>();
+
+ for (Map.Entry<String, LocalResource> rsrc : localResourceMap.entrySet()) {
+ LocalResource resource = rsrc.getValue();
+ LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
+ allResources.putIfAbsent(req, new ArrayList<>());
+ allResources.get(req).add(rsrc.getKey());
+ storeSharedCacheUploadPolicy(req,
+ resource.getShouldBeUploadedToSharedCache());
+ switch (resource.getVisibility()) {
+ case PUBLIC:
+ publicList.add(req);
+ break;
+ case PRIVATE:
+ privateList.add(req);
+ break;
+ case APPLICATION:
+ appList.add(req);
+ break;
+ default:
+ break;
+ }
+ }
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ new LinkedHashMap<>();
+ if (!publicList.isEmpty()) {
+ publicRsrcs.addAll(publicList);
+ req.put(LocalResourceVisibility.PUBLIC, publicList);
+ }
+ if (!privateList.isEmpty()) {
+ privateRsrcs.addAll(privateList);
+ req.put(LocalResourceVisibility.PRIVATE, privateList);
+ }
+ if (!appList.isEmpty()) {
+ appRsrcs.addAll(appList);
+ req.put(LocalResourceVisibility.APPLICATION, appList);
+ }
+ if (!allResources.isEmpty()) {
+ this.pendingResources.putAll(allResources);
+ }
+ return req;
+ }
+
+ /**
+ * Called when resource localized.
+ * @param request The original request for the localized resource
+ * @param location The path where the resource is localized
+ * @return The list of symlinks for the localized resources.
+ */
+ public List<String> resourceLocalized(LocalResourceRequest request,
+ Path location) {
+ List<String> symlinks = pendingResources.remove(request);
+ if (symlinks == null) {
+ return null;
+ } else {
+ localizedResources.put(location, symlinks);
+ return symlinks;
+ }
+ }
+
+ public void resourceLocalizationFailed(LocalResourceRequest request) {
+ pendingResources.remove(request);
+ resourcesFailedToBeLocalized.add(request);
+ }
+
+ public synchronized Map<LocalResourceVisibility,
+ Collection<LocalResourceRequest>> getAllResourcesByVisibility() {
+
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
+ new HashMap<>();
+ if (!publicRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
+ }
+ if (!privateRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
+ }
+ if (!appRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
+ }
+ return rsrc;
+ }
+
+ /**
+ * Store the resource's shared cache upload policies
+ * Given LocalResourceRequest can be shared across containers in
+ * LocalResourcesTrackerImpl, we preserve the upload policies here.
+ * In addition, it is possible for the application to create several
+ * "identical" LocalResources as part of
+ * ContainerLaunchContext.setLocalResources with different symlinks.
+ * There is a corner case where these "identical" local resources have
+ * different upload policies. For that scenario, upload policy will be set to
+ * true as long as there is at least one LocalResource entry with
+ * upload policy set to true.
+ */
+ private void storeSharedCacheUploadPolicy(
+ LocalResourceRequest resourceRequest, Boolean uploadPolicy) {
+ Boolean storedUploadPolicy = resourcesUploadPolicies.get(resourceRequest);
+ if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) {
+ resourcesUploadPolicies.put(resourceRequest, uploadPolicy);
+ }
+ }
+
+ public Map<Path, List<String>> getLocalizedResources() {
+ return localizedResources;
+ }
+
+ public Map<LocalResourceRequest, Path> getResourcesToBeUploaded() {
+ return resourcesToBeUploaded;
+ }
+
+ public Map<LocalResourceRequest, Boolean> getResourcesUploadPolicies() {
+ return resourcesUploadPolicies;
+ }
+
+ public Map<LocalResourceRequest, List<String>> getPendingResources() {
+ return pendingResources;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
index 11bb25e..43a2f33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
@@ -44,7 +44,7 @@ public class ContainerLocalizationRequestEvent extends
*/
public ContainerLocalizationRequestEvent(Container c,
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc) {
- super(LocalizationEventType.INIT_CONTAINER_RESOURCES, c);
+ super(LocalizationEventType.LOCALIZE_CONTAINER_RESOURCES, c);
this.rsrc = rsrc;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
index 4785fba..cb9842d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.eve
public enum LocalizationEventType {
INIT_APPLICATION_RESOURCES,
- INIT_CONTAINER_RESOURCES,
+ LOCALIZE_CONTAINER_RESOURCES,
CACHE_CLEANUP,
CLEANUP_CONTAINER_RESOURCES,
DESTROY_APPLICATION_RESOURCES,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 0371863..8e68889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -326,7 +326,7 @@ public class NMTimelinePublisher extends CompositeService {
publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
ContainerMetricsConstants.LOCALIZATION_FINISHED_EVENT_TYPE);
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
ContainerMetricsConstants.LOCALIZATION_START_EVENT_TYPE);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index 4aeb2e2..740ed19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -86,7 +86,7 @@ public class DummyContainerManager extends ContainerManagerImpl {
dispatcher.getEventHandler().handle(new ApplicationInitedEvent(
app.getAppId()));
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
ContainerLocalizationRequestEvent rsrcReqs =
(ContainerLocalizationRequestEvent) event;
// simulate localization of all requested resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 3e00885..aa0d975 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -131,7 +131,18 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
LOG.info("Running testContainerLaunchAndExitFailure");
super.testContainerLaunchAndExitFailure();
}
-
+
+ @Override
+ public void testLocalingResourceWhileContainerRunning()
+ throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ super.testLocalingResourceWhileContainerRunning();
+ }
+
@Override
public void testLocalFilesCleanup() throws InterruptedException,
IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index d8660dd..031300f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor;
@@ -715,6 +716,11 @@ public abstract class BaseAMRMProxyTest {
@Override
public NMTimelinePublisher getNMTimelinePublisher() {
+ return null;
+ }
+
+ @Override
+ public ContainerExecutor getContainerExecutor() {
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 1f803b4..5785e1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -36,17 +36,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Supplier;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -83,6 +86,7 @@ import org.apache.hadoop.yarn.server.nodemanager.CMgrSignalContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestAuxServices.ServiceA;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -459,7 +463,138 @@ public class TestContainerManager extends BaseContainerManagerTest {
// and verify exit code returned
testContainerLaunchAndExit(exitCode);
}
-
+
+ private Map<String, LocalResource> setupLocalResources(String fileName,
+ String symLink) throws Exception {
+ // ////// Create the resources for the container
+ File dir = new File(tmpDir, "dir");
+ dir.mkdirs();
+ File file = new File(dir, fileName);
+ PrintWriter fileWriter = new PrintWriter(file);
+ fileWriter.write("Hello World!");
+ fileWriter.close();
+
+ URL resourceURL = URL.fromPath(FileContext.getLocalFSFileContext()
+ .makeQualified(new Path(file.getAbsolutePath())));
+ LocalResource resource =
+ recordFactory.newRecordInstance(LocalResource.class);
+ resource.setResource(resourceURL);
+ resource.setSize(-1);
+ resource.setVisibility(LocalResourceVisibility.APPLICATION);
+ resource.setType(LocalResourceType.FILE);
+ resource.setTimestamp(file.lastModified());
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(symLink, resource);
+ return localResources;
+ }
+
+ // Start the container
+ // While the container is running, localize new resources.
+ // Verify the symlink is created properly
+ @Test
+ public void testLocalingResourceWhileContainerRunning() throws Exception {
+ // Real del service
+ delSrvc = new DeletionService(exec);
+ delSrvc.init(conf);
+
+ ((NodeManager.NMContext)context).setContainerExecutor(exec);
+ containerManager = createContainerManager(delSrvc);
+ containerManager.init(conf);
+ containerManager.start();
+ // set up local resources
+ Map<String, LocalResource> localResource =
+ setupLocalResources("file", "symLink1");
+ ContainerLaunchContext context =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ context.setLocalResources(localResource);
+
+ // a long running container - sleep
+ context.setCommands(Arrays.asList("sleep 6"));
+ ContainerId cId = createContainerId(0);
+
+ // start the container
+ StartContainerRequest scRequest = StartContainerRequest.newInstance(context,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, this.context.getNodeId(),
+ user, this.context.getContainerTokenSecretManager()));
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(Arrays.asList(scRequest));
+ containerManager.startContainers(allRequests);
+ BaseContainerManagerTest
+ .waitForContainerState(containerManager, cId, ContainerState.RUNNING);
+
+ BaseContainerManagerTest.waitForApplicationState(containerManager,
+ cId.getApplicationAttemptId().getApplicationId(),
+ ApplicationState.RUNNING);
+ checkResourceLocalized(cId, "symLink1");
+
+ // Localize new local resources while container is running
+ Map<String, LocalResource> localResource2 =
+ setupLocalResources("file2", "symLink2");
+
+ ResourceLocalizationRequest request =
+ ResourceLocalizationRequest.newInstance(cId, localResource2);
+ containerManager.localize(request);
+
+ // Verify resource is localized and symlink is created.
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ try {
+ checkResourceLocalized(cId, "symLink2");
+ return true;
+ } catch (Throwable e) {
+ return false;
+ }
+ }
+ }, 500, 20000);
+
+ BaseContainerManagerTest
+ .waitForContainerState(containerManager, cId, ContainerState.COMPLETE);
+ // Verify container cannot localize resources while at non-running state.
+ try{
+ containerManager.localize(request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().contains("Not able to localize new resources"));
+ }
+ }
+
+ private void checkResourceLocalized(ContainerId containerId, String symLink) {
+ String appId =
+ containerId.getApplicationAttemptId().getApplicationId().toString();
+ File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
+ File userDir = new File(userCacheDir, user);
+ File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
+ // localDir/usercache/nobody/appcache/application_0_0000
+ File appDir = new File(appCache, appId);
+ // localDir/usercache/nobody/appcache/application_0_0000/container_0_0000_01_000000
+ File containerDir = new File(appDir, containerId.toString());
+ // localDir/usercache/nobody/appcache/application_0_0000/container_0_0000_01_000000/symLink1
+ File targetFile = new File(containerDir, symLink);
+
+ File sysDir =
+ new File(localDir, ResourceLocalizationService.NM_PRIVATE_DIR);
+ // localDir/nmPrivate/application_0_0000
+ File appSysDir = new File(sysDir, appId);
+ // localDir/nmPrivate/application_0_0000/container_0_0000_01_000000
+ File containerSysDir = new File(appSysDir, containerId.toString());
+
+ Assert.assertTrue("AppDir " + appDir.getAbsolutePath() + " doesn't exist!!",
+ appDir.exists());
+ Assert.assertTrue(
+ "AppSysDir " + appSysDir.getAbsolutePath() + " doesn't exist!!",
+ appSysDir.exists());
+ Assert.assertTrue(
+ "containerDir " + containerDir.getAbsolutePath() + " doesn't exist !",
+ containerDir.exists());
+ Assert.assertTrue("containerSysDir " + containerSysDir.getAbsolutePath()
+ + " doesn't exist !", containerDir.exists());
+ Assert.assertTrue(
+ "targetFile " + targetFile.getAbsolutePath() + " doesn't exist !!",
+ targetFile.exists());
+ }
+
@Test
public void testLocalFilesCleanup() throws InterruptedException,
IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 23786fe..7aa657e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -66,6 +66,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.junit.Assert;
@@ -1767,7 +1768,7 @@ public class TestResourceLocalizationService {
// creating new containers and populating corresponding localizer runners
// Container - 1
- ContainerImpl container1 = createMockContainer(user, 1);
+ Container container1 = createMockContainer(user, 1);
String localizerId1 = container1.getContainerId().toString();
rls.getPrivateLocalizers().put(
localizerId1,
@@ -2292,7 +2293,7 @@ public class TestResourceLocalizationService {
}
private ContainerLocalizationRequestEvent createContainerLocalizationEvent(
- ContainerImpl container, LocalResourceVisibility vis,
+ Container container, LocalResourceVisibility vis,
LocalResourceRequest req) {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> reqs =
new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
@@ -2310,6 +2311,7 @@ public class TestResourceLocalizationService {
when(container.getUser()).thenReturn(user);
Credentials mockCredentials = mock(Credentials.class);
when(container.getCredentials()).thenReturn(mockCredentials);
+ when(container.getContainerState()).thenReturn(ContainerState.LOCALIZING);
return container;
}
@@ -2358,6 +2360,7 @@ public class TestResourceLocalizationService {
creds.addToken(new Text("tok" + id), tk);
when(c.getCredentials()).thenReturn(creds);
when(c.toString()).thenReturn(cId.toString());
+ when(c.getContainerState()).thenReturn(ContainerState.LOCALIZING);
return c;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 2df0c98..d24f89d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -89,6 +89,13 @@ public class TestContainersMonitorResourceChange {
public void deleteAsUser(DeletionAsUserContext ctx)
throws IOException, InterruptedException {
}
+
+ @Override
+ public void symLink(String target, String symlink)
+ throws IOException {
+
+ }
+
@Override
public String getProcessId(ContainerId containerId) {
return String.valueOf(containerId.getContainerId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 8332b2a..c176556 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -18,11 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
@@ -41,8 +37,14 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
public class MockContainer implements Container {
private ContainerId id;
@@ -119,6 +121,11 @@ public class MockContainer implements Container {
}
@Override
+ public ResourceSet getResourceSet() {
+ return null;
+ }
+
+ @Override
public void handle(ContainerEvent event) {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/43] hadoop git commit: HDFS-10841. Remove duplicate or unused
variable in appendFile(). Contributed by Kihwal Lee.
Posted by ae...@apache.org.
HDFS-10841. Remove duplicate or unused variable in appendFile(). Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6c0b754
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6c0b754
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6c0b754
Branch: refs/heads/HDFS-7240
Commit: f6c0b7543f612de756ff0c03e9a2c6e33b496a36
Parents: 7c213c3
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 11:48:35 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 11:48:35 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c0b754/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 5192352..6f898ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -84,12 +84,10 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
- final String src;
final INodesInPath iip;
fsd.writeLock();
try {
iip = fsd.resolvePathForWrite(pc, srcArg);
- src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
final String path = iip.getPath();
@@ -111,7 +109,7 @@ final class FSDirAppendOp {
// not support appending file with striped blocks
if (file.isStriped()) {
throw new UnsupportedOperationException(
- "Cannot append to files with striped block " + src);
+ "Cannot append to files with striped block " + path);
}
BlockManager blockManager = fsd.getBlockManager();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/43] hadoop git commit: HDFS-9781. FsDatasetImpl#getBlockReports
can occasionally throw NullPointerException. Contributed by Manoj
Govindassamy.
Posted by ae...@apache.org.
HDFS-9781. FsDatasetImpl#getBlockReports can occasionally throw NullPointerException. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07650bc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07650bc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07650bc3
Branch: refs/heads/HDFS-7240
Commit: 07650bc37a3c78ecc6566d813778d0954d0b06b0
Parents: f6ea9be
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Sep 2 15:26:20 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Sep 2 15:33:11 2016 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 21 +++++++--
.../fsdataset/impl/TestFsDatasetImpl.java | 49 ++++++++++++++++----
2 files changed, 56 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07650bc3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 129024b..e0d2baf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1827,13 +1827,24 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
Map<String, BlockListAsLongs.Builder> builders =
new HashMap<String, BlockListAsLongs.Builder>();
- List<FsVolumeImpl> curVolumes = volumes.getVolumes();
- for (FsVolumeSpi v : curVolumes) {
- builders.put(v.getStorageID(), BlockListAsLongs.builder(maxDataLength));
- }
-
+ List<FsVolumeImpl> curVolumes = null;
try (AutoCloseableLock lock = datasetLock.acquire()) {
+ curVolumes = volumes.getVolumes();
+ for (FsVolumeSpi v : curVolumes) {
+ builders.put(v.getStorageID(), BlockListAsLongs.builder(maxDataLength));
+ }
+
+ Set<String> missingVolumesReported = new HashSet<>();
for (ReplicaInfo b : volumeMap.replicas(bpid)) {
+ String volStorageID = b.getVolume().getStorageID();
+ if (!builders.containsKey(volStorageID)) {
+ if (!missingVolumesReported.contains(volStorageID)) {
+ LOG.warn("Storage volume: " + volStorageID + " missing for the"
+ + " replica block: " + b + ". Probably being removed!");
+ missingVolumesReported.add(volStorageID);
+ }
+ continue;
+ }
switch(b.getState()) {
case FINALIZED:
case RBW:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07650bc3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index e73a612..b946803 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -590,8 +590,15 @@ public class TestFsDatasetImpl {
final ExtendedBlock eb = new ExtendedBlock(BLOCK_POOL_IDS[0], 0);
final CountDownLatch startFinalizeLatch = new CountDownLatch(1);
final CountDownLatch brReceivedLatch = new CountDownLatch(1);
+ final CountDownLatch volRemovedLatch = new CountDownLatch(1);
class BlockReportThread extends Thread {
public void run() {
+ // Lets wait for the volume remove process to start
+ try {
+ volRemovedLatch.await();
+ } catch (Exception e) {
+ LOG.info("Unexpected exception when waiting for vol removal:", e);
+ }
LOG.info("Getting block report");
dataset.getBlockReports(eb.getBlockPoolId());
LOG.info("Successfully received block report");
@@ -599,18 +606,27 @@ public class TestFsDatasetImpl {
}
}
- final BlockReportThread brt = new BlockReportThread();
class ResponderThread extends Thread {
public void run() {
try (ReplicaHandler replica = dataset
.createRbw(StorageType.DEFAULT, eb, false)) {
- LOG.info("createRbw finished");
+ LOG.info("CreateRbw finished");
startFinalizeLatch.countDown();
- // Slow down while we're holding the reference to the volume
- Thread.sleep(1000);
+ // Slow down while we're holding the reference to the volume.
+ // As we finalize a block, the volume is removed in parallel.
+ // Ignore any interrupts coming out of volume shutdown.
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ie) {
+ LOG.info("Ignoring ", ie);
+ }
+
+ // Lets wait for the other thread finish getting block report
+ brReceivedLatch.await();
+
dataset.finalizeBlock(eb);
- LOG.info("finalizeBlock finished");
+ LOG.info("FinalizeBlock finished");
} catch (Exception e) {
LOG.warn("Exception caught. This should not affect the test", e);
}
@@ -621,13 +637,28 @@ public class TestFsDatasetImpl {
res.start();
startFinalizeLatch.await();
+ // Verify if block report can be received
+ // when volume is being removed
+ final BlockReportThread brt = new BlockReportThread();
+ brt.start();
+
Set<File> volumesToRemove = new HashSet<>();
volumesToRemove.add(
StorageLocation.parse(dataset.getVolume(eb).getBasePath()).getFile());
- LOG.info("Removing volume " + volumesToRemove);
- // Verify block report can be received during this
- brt.start();
- dataset.removeVolumes(volumesToRemove, true);
+ /**
+ * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing
+ * IllegalMonitorStateException when there is a parallel reader/writer
+ * to the volume. Remove below try/catch block after fixing HDFS-10830.
+ */
+ try {
+ LOG.info("Removing volume " + volumesToRemove);
+ dataset.removeVolumes(volumesToRemove, true);
+ } catch (Exception e) {
+ LOG.info("Unexpected issue while removing volume: ", e);
+ } finally {
+ volRemovedLatch.countDown();
+ }
+
LOG.info("Volumes removed");
brReceivedLatch.await();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/43] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f388a209/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
new file mode 100644
index 0000000..edf47ca
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
@@ -0,0 +1,1854 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" 3.0.0-alpha1 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HDFS-46](https://issues.apache.org/jira/browse/HDFS-46) | *Major* | **The namespace quota of root directory should not be Integer.MAX\_VALUE**
+
+Change default namespace quota of root directory from Integer.MAX\_VALUE to Long.MAX\_VALUE.
+
+
+---
+
+* [HADOOP-8124](https://issues.apache.org/jira/browse/HADOOP-8124) | *Major* | **Remove the deprecated Syncable.sync() method**
+
+Remove the deprecated FSDataOutputStream constructor, FSDataOutputStream.sync() and Syncable.sync().
+
+
+---
+
+* [HDFS-3034](https://issues.apache.org/jira/browse/HDFS-3034) | *Major* | **Remove the deprecated Syncable.sync() method**
+
+Remove the deprecated DFSOutputStream.sync() method.
+
+
+---
+
+* [HADOOP-7659](https://issues.apache.org/jira/browse/HADOOP-7659) | *Minor* | **fs -getmerge isn't guaranteed to work well over non-HDFS filesystems**
+
+Documented that the "fs -getmerge" shell command may not work properly over non HDFS-filesystem implementations due to platform-varying file list ordering.
+
+
+---
+
+* [HADOOP-8776](https://issues.apache.org/jira/browse/HADOOP-8776) | *Minor* | **Provide an option in test-patch that can enable / disable compiling native code**
+
+test-patch.sh adds a new option "--build-native". When set to false native
+components are not built. When set to true native components are built. The
+default value is true.
+
+
+---
+
+* [HDFS-5079](https://issues.apache.org/jira/browse/HDFS-5079) | *Major* | **Cleaning up NNHAStatusHeartbeat.State DatanodeProtocolProtos.**
+
+This change affects wire-compatibility of the NameNode/DataNode heartbeat protocol.
+
+
+---
+
+* [HDFS-5570](https://issues.apache.org/jira/browse/HDFS-5570) | *Major* | **Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs**
+
+Support for hftp and hsftp has been removed. They have superseded by webhdfs and swebhdfs.
+
+
+---
+
+* [HADOOP-10485](https://issues.apache.org/jira/browse/HADOOP-10485) | *Major* | **Remove dead classes in hadoop-streaming**
+
+Deprecated and unused classes in the org.apache.hadoop.record package have been removed from hadoop-streaming.
+
+
+---
+
+* [HDFS-6246](https://issues.apache.org/jira/browse/HDFS-6246) | *Minor* | **Remove 'dfs.support.append' flag from trunk code**
+
+Appends in HDFS can no longer be disabled.
+
+
+---
+
+* [HADOOP-10474](https://issues.apache.org/jira/browse/HADOOP-10474) | *Major* | **Move o.a.h.record to hadoop-streaming**
+
+The classes in org.apache.hadoop.record are moved from hadoop-common to a new hadoop-streaming artifact within the hadoop-tools module.
+
+
+---
+
+* [HDFS-2538](https://issues.apache.org/jira/browse/HDFS-2538) | *Minor* | **option to disable fsck dots**
+
+fsck does not print out dots for progress reporting by default. To print out dots, you should specify '-showprogress' option.
+
+
+---
+
+* [HADOOP-9902](https://issues.apache.org/jira/browse/HADOOP-9902) | *Major* | **Shell script rewrite**
+
+<!-- markdown -->
+The Hadoop shell scripts have been rewritten to fix many long standing bugs and include some new features. While an eye has been kept towards compatibility, some changes may break existing installations.
+
+INCOMPATIBLE CHANGES:
+
+* The pid and out files for secure daemons have been renamed to include the appropriate ${HADOOP\_IDENT\_STR}. This should allow, with proper configurations in place, for multiple versions of the same secure daemon to run on a host. Additionally, pid files are now created when daemons are run in interactive mode. This will also prevent the accidental starting of two daemons with the same configuration prior to launching java (i.e., "fast fail" without having to wait for socket opening).
+* All Hadoop shell script subsystems now execute hadoop-env.sh, which allows for all of the environment variables to be in one location. This was not the case previously.
+* The default content of *-env.sh has been significantly altered, with the majority of defaults moved into more protected areas inside the code. Additionally, these files do not auto-append anymore; setting a variable on the command line prior to calling a shell command must contain the entire content, not just any extra settings. This brings Hadoop more in-line with the vast majority of other software packages.
+* All HDFS\_\*, YARN\_\*, and MAPRED\_\* environment variables act as overrides to their equivalent HADOOP\_\* environment variables when 'hdfs', 'yarn', 'mapred', and related commands are executed. Previously, these were separated out which meant a significant amount of duplication of common settings.
+* hdfs-config.sh and hdfs-config.cmd were inadvertently duplicated into libexec and sbin. The sbin versions have been removed.
+* The log4j settings forcibly set by some *-daemon.sh commands have been removed. These settings are now configurable in the \*-env.sh files via \*\_OPT.
+* Support for various undocumented YARN log4j.properties files has been removed.
+* Support for ${HADOOP\_MASTER} and the related rsync code have been removed.
+* The undocumented and unused yarn.id.str Java property has been removed.
+* The unused yarn.policy.file Java property has been removed.
+* We now require bash v3 (released July 27, 2004) or better in order to take advantage of better regex handling and ${BASH\_SOURCE}. POSIX sh will not work.
+* Support for --script has been removed. We now use ${HADOOP\_\*\_PATH} or ${HADOOP\_PREFIX} to find the necessary binaries. (See other note regarding ${HADOOP\_PREFIX} auto discovery.)
+* Non-existent classpaths, ld.so library paths, JNI library paths, etc, will be ignored and stripped from their respective environment settings.
+
+NEW FEATURES:
+
+* Daemonization has been moved from *-daemon.sh to the bin commands via the --daemon option. Simply use --daemon start to start a daemon, --daemon stop to stop a daemon, and --daemon status to set $? to the daemon's status. The return code for status is LSB-compatible. For example, 'hdfs --daemon start namenode'.
+* It is now possible to override some of the shell code capabilities to provide site specific functionality without replacing the shipped versions. Replacement functions should go into the new hadoop-user-functions.sh file.
+* A new option called --buildpaths will attempt to add developer build directories to the classpath to allow for in source tree testing.
+* Operations which trigger ssh connections can now use pdsh if installed. ${HADOOP\_SSH\_OPTS} still gets applied.
+* Added distch and jnipath subcommands to the hadoop command.
+* Shell scripts now support a --debug option which will report basic information on the construction of various environment variables, java options, classpath, etc. to help in configuration debugging.
+
+BUG FIXES:
+
+* ${HADOOP\_CONF\_DIR} is now properly honored everywhere, without requiring symlinking and other such tricks.
+* ${HADOOP\_CONF\_DIR}/hadoop-layout.sh is now documented with a provided hadoop-layout.sh.example file.
+* Shell commands should now work properly when called as a relative path, without ${HADOOP\_PREFIX} being defined, and as the target of bash -x for debugging. If ${HADOOP\_PREFIX} is not set, it will be automatically determined based upon the current location of the shell library. Note that other parts of the extended Hadoop ecosystem may still require this environment variable to be configured.
+* Operations which trigger ssh will now limit the number of connections to run in parallel to ${HADOOP\_SSH\_PARALLEL} to prevent memory and network exhaustion. By default, this is set to 10.
+* ${HADOOP\_CLIENT\_OPTS} support has been added to a few more commands.
+* Some subcommands were not listed in the usage.
+* Various options on hadoop command lines were supported inconsistently. These have been unified into hadoop-config.sh. --config is still required to be first, however.
+* ulimit logging for secure daemons no longer assumes /bin/bash but does assume bash is on the command line path.
+* Removed references to some Yahoo! specific paths.
+* Removed unused slaves.sh from YARN build tree.
+* Many exit states have been changed to reflect reality.
+* Shell level errors now go to STDERR. Before, many of them went incorrectly to STDOUT.
+* CDPATH with a period (.) should no longer break the scripts.
+* The scripts no longer try to chown directories.
+* If ${JAVA\_HOME} is not set on OS X, it now properly detects it instead of throwing an error.
+
+IMPROVEMENTS:
+
+* The *.out files are now appended instead of overwritten to allow for external log rotation.
+* The style and layout of the scripts is much more consistent across subprojects.
+* More of the shell code is now commented.
+* Significant amounts of redundant code have been moved into a new file called hadoop-functions.sh.
+* The various *-env.sh have been massively changed to include documentation and examples on what can be set, ramifications of setting, etc. for all variables that are expected to be set by a user.
+* There is now some trivial de-duplication and sanitization of the classpath and JVM options. This allows, amongst other things, for custom settings in \*\_OPTS for Hadoop daemons to override defaults and other generic settings (i.e., ${HADOOP\_OPTS}). This is particularly relevant for Xmx settings, as one can now set them in _OPTS and ignore the heap specific options for daemons which force the size in megabytes.
+* Subcommands have been alphabetized in both usage and in the code.
+* All/most of the functionality provided by the sbin/* commands has been moved to either their bin/ equivalents or made into functions. The rewritten versions of these commands are now wrappers to maintain backward compatibility.
+* Usage information is given with the following options/subcommands for all scripts using the common framework: --? -? ? --help -help -h help
+* Several generic environment variables have been added to provide a common configuration for pids, logs, and their security equivalents. The older versions still act as overrides to these generic versions.
+* Groundwork has been laid to allow for custom secure daemon setup using something other than jsvc (e.g., pfexec on Solaris).
+* Scripts now test and report better error messages for various states of the log and pid dirs on daemon startup. Before, unprotected shell errors would be displayed to the user.
+
+
+---
+
+* [HADOOP-11041](https://issues.apache.org/jira/browse/HADOOP-11041) | *Minor* | **VersionInfo output specifies subversion**
+
+This changes the output of the 'hadoop version' command to generically say 'Source code repository' rather than specify which type of repo.
+
+
+---
+
+* [MAPREDUCE-5972](https://issues.apache.org/jira/browse/MAPREDUCE-5972) | *Trivial* | **Fix typo 'programatically' in job.xml (and a few other places)**
+
+Fix a typo. If a configuration is set through program, the source of the configuration is set to 'programmatically' instead of 'programatically' now.
+
+
+---
+
+* [MAPREDUCE-2841](https://issues.apache.org/jira/browse/MAPREDUCE-2841) | *Major* | **Task level native optimization**
+
+Adds a native implementation of the map output collector. The native library will build automatically with -Pnative. Users may choose the new collector on a job-by-job basis by setting mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.
+nativetask.NativeMapOutputCollectorDelegator in their job configuration. For shuffle-intensive jobs this may provide speed-ups of 30% or more.
+
+
+---
+
+* [HADOOP-11356](https://issues.apache.org/jira/browse/HADOOP-11356) | *Major* | **Removed deprecated o.a.h.fs.permission.AccessControlException**
+
+org.apache.hadoop.fs.permission.AccessControlException was deprecated in the last major release, and has been removed in favor of org.apache.hadoop.security.AccessControlException
+
+
+---
+
+* [HADOOP-10950](https://issues.apache.org/jira/browse/HADOOP-10950) | *Major* | **rework heap management vars**
+
+<!-- markdown -->
+
+* HADOOP\_HEAPSIZE variable has been deprecated (It will still be honored if set, but expect it to go away in the future). In its place, HADOOP\_HEAPSIZE\_MAX and HADOOP\_HEAPSIZE\_MIN have been introduced to set Xmx and Xms, respectively.
+
+* The internal variable JAVA\_HEAP\_MAX has been removed.
+
+* Default heap sizes have been removed. This will allow for the JVM to use auto-tuning based upon the memory size of the host. To re-enable the old default, configure HADOOP\_HEAPSIZE_MAX="1g" in hadoop-env.sh.
+
+* All global and daemon-specific heap size variables now support units. If the variable is only a number, the size is assumed to be in megabytes.
+
+
+---
+
+* [HADOOP-11353](https://issues.apache.org/jira/browse/HADOOP-11353) | *Major* | **Add support for .hadooprc**
+
+.hadooprc allows users a convenient way to set and/or override the shell level settings.
+
+
+---
+
+* [MAPREDUCE-5785](https://issues.apache.org/jira/browse/MAPREDUCE-5785) | *Major* | **Derive heap size or mapreduce.\*.memory.mb automatically**
+
+The memory values for mapreduce.map/reduce.memory.mb keys, if left to their default values of -1, will now be automatically inferred from the heap size value system property (-Xmx) specified for mapreduce.map/reduce.java.opts keys.
+
+The converse is also done, i.e. if mapreduce.map/reduce.memory.mb values are specified, but no -Xmx is supplied for mapreduce.map/reduce.java.opts keys, then the -Xmx value will be derived from the former's value.
+
+If neither is specified, then a default value of 1024 MB gets used.
+
+For both these conversions, a scaling factor specified by property mapreduce.job.heap.memory-mb.ratio is used, to account for overheads between heap usage vs. actual physical memory usage.
+
+Existing configs or job code that already specify both the set of properties explicitly would not be affected by this inferring change.
+
+
+---
+
+* [YARN-2428](https://issues.apache.org/jira/browse/YARN-2428) | *Trivial* | **LCE default banned user list should have yarn**
+
+The user 'yarn' is no longer allowed to run tasks for security reasons.
+
+
+---
+
+* [HADOOP-11460](https://issues.apache.org/jira/browse/HADOOP-11460) | *Major* | **Deprecate shell vars**
+
+<!-- markdown -->
+
+| Old | New |
+|:---- |:---- |
+| HADOOP\_HDFS\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| HADOOP\_HDFS\_LOGFILE | HADOOP\_LOGFILE |
+| HADOOP\_HDFS\_NICENESS | HADOOP\_NICENESS |
+| HADOOP\_HDFS\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| HADOOP\_HDFS\_PID\_DIR | HADOOP\_PID\_DIR |
+| HADOOP\_HDFS\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| HADOOP\_HDFS\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| HADOOP\_MAPRED\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| HADOOP\_MAPRED\_LOGFILE | HADOOP\_LOGFILE |
+| HADOOP\_MAPRED\_NICENESS | HADOOP\_NICENESS |
+| HADOOP\_MAPRED\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| HADOOP\_MAPRED\_PID\_DIR | HADOOP\_PID\_DIR |
+| HADOOP\_MAPRED\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| HADOOP\_MAPRED\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| YARN\_CONF\_DIR | HADOOP\_CONF\_DIR |
+| YARN\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| YARN\_LOGFILE | HADOOP\_LOGFILE |
+| YARN\_NICENESS | HADOOP\_NICENESS |
+| YARN\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| YARN\_PID\_DIR | HADOOP\_PID\_DIR |
+| YARN\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| YARN\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| YARN\_OPTS | HADOOP\_OPTS |
+| YARN\_SLAVES | HADOOP\_SLAVES |
+| YARN\_USER\_CLASSPATH | HADOOP\_CLASSPATH |
+| YARN\_USER\_CLASSPATH\_FIRST | HADOOP\_USER\_CLASSPATH\_FIRST |
+| KMS\_CONFIG | HADOOP\_CONF\_DIR |
+| KMS\_LOG | HADOOP\_LOG\_DIR |
+
+
+---
+
+* [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | *Trivial* | **dfs -count -q should label output column**
+
+Added -v option to fs -count command to display a header record in the report.
+
+
+---
+
+* [HADOOP-11485](https://issues.apache.org/jira/browse/HADOOP-11485) | *Major* | **Pluggable shell integration**
+
+Support for shell profiles has been added. They allow for easy integration of additional functionality, classpaths, and more from inside the bash scripts rather than relying upon modifying hadoop-env.sh, etc. See the Unix Shell Guide for more information.
+
+
+---
+
+* [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | *Minor* | **Shell command ls should include sort options**
+
+Options to sort output of fs -ls comment: -t (mtime), -S (size), -u (atime), -r (reverse)
+
+
+---
+
+* [HADOOP-11554](https://issues.apache.org/jira/browse/HADOOP-11554) | *Major* | **Expose HadoopKerberosName as a hadoop subcommand**
+
+The hadoop kerbname subcommand has been added to ease operational pain in determining the output of auth\_to\_local rules.
+
+
+---
+
+* [HDFS-7460](https://issues.apache.org/jira/browse/HDFS-7460) | *Major* | **Rewrite httpfs to use new shell framework**
+
+<!-- markdown -->
+This deprecates the following environment variables:
+
+| Old | New |
+|:---- |:---- |
+| HTTPFS_LOG | HADOOP_LOG_DIR|
+| HTTPFS_CONFG | HADOOP_CONF_DIR |
+
+
+---
+
+* [MAPREDUCE-5653](https://issues.apache.org/jira/browse/MAPREDUCE-5653) | *Major* | **DistCp does not honour config-overrides for mapreduce.[map,reduce].memory.mb**
+
+Prior to this change, distcp had hard-coded values for memory usage. Now distcp will honor memory settings in a way compatible with the rest of MapReduce.
+
+
+---
+
+* [HADOOP-11657](https://issues.apache.org/jira/browse/HADOOP-11657) | *Minor* | **Align the output of `hadoop fs -du` to be more Unix-like**
+
+The output of du has now been made more Unix-like, with aligned output.
+
+
+---
+
+* [HDFS-7302](https://issues.apache.org/jira/browse/HDFS-7302) | *Major* | **namenode -rollingUpgrade downgrade may finalize a rolling upgrade**
+
+Remove "downgrade" from "namenode -rollingUpgrade" startup option since it may incorrectly finalize an ongoing rolling upgrade.
+
+
+---
+
+* [HADOOP-6857](https://issues.apache.org/jira/browse/HADOOP-6857) | *Major* | **FsShell should report raw disk usage including replication factor**
+
+The output format of hadoop fs -du has been changed. It shows not only the file size but also the raw disk usage including the replication factor.
+
+
+---
+
+* [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | *Major* | **Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY**
+
+Use low latency TCP connections for hadoop IPC
+
+
+---
+
+* [HADOOP-10115](https://issues.apache.org/jira/browse/HADOOP-10115) | *Major* | **Exclude duplicate jars in hadoop package under different component's lib**
+
+Jars in the various subproject lib directories are now de-duplicated against Hadoop common. Users who interact directly with those directories must be sure to pull in common's dependencies as well.
+
+
+---
+
+* [YARN-3154](https://issues.apache.org/jira/browse/YARN-3154) | *Blocker* | **Should not upload partial logs for MR jobs or other "short-running' applications**
+
+Applications which made use of the LogAggregationContext in their application will need to revisit this code in order to make sure that their logs continue to get rolled out.
+
+
+---
+
+* [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | *Major* | **Add posixGroups support for LDAP groups mapping service**
+
+Add posixGroups support for LDAP groups mapping service. The change in LDAPGroupMapping is compatible with previous scenario. In LDAP, the group mapping between {{posixAccount}} and {{posixGroup}} is different from the general LDAPGroupMapping, one of the differences is the {{"memberUid"}} will be used to mapping {{posixAccount}} and {{posixGroup}}. The feature will handle the mapping in internal when configuration {{hadoop.security.group.mapping.ldap.search.filter.user}} is set as "posixAccount" and {{hadoop.security.group.mapping.ldap.search.filter.group}} is "posixGroup".
+
+
+---
+
+* [MAPREDUCE-4424](https://issues.apache.org/jira/browse/MAPREDUCE-4424) | *Minor* | **'mapred job -list' command should show the job name as well**
+
+Now "mapred job -list" command displays the Job Name as well.
+
+
+---
+
+* [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | *Major* | **FairScheduler handles "invalid" queue names inconsistently**
+
+FairScheduler does not allow queue names with leading or tailing spaces or empty sub-queue names anymore.
+
+
+---
+
+* [HDFS-7985](https://issues.apache.org/jira/browse/HDFS-7985) | *Major* | **WebHDFS should be always enabled**
+
+WebHDFS is mandatory and cannot be disabled.
+
+
+---
+
+* [HDFS-6353](https://issues.apache.org/jira/browse/HDFS-6353) | *Major* | **Check and make checkpoint before stopping the NameNode**
+
+Stopping the namenode on secure systems now requires the user be authenticated.
+
+
+---
+
+* [HADOOP-11553](https://issues.apache.org/jira/browse/HADOOP-11553) | *Blocker* | **Formalize the shell API**
+
+Python is now required to build the documentation.
+
+
+---
+
+* [HDFS-7501](https://issues.apache.org/jira/browse/HDFS-7501) | *Major* | **TransactionsSinceLastCheckpoint can be negative on SBNs**
+
+Fixed a bug where the StandbyNameNode's TransactionsSinceLastCheckpoint metric may slide into a negative number after every subsequent checkpoint.
+
+
+---
+
+* [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | *Minor* | **Add support for hardware crc of HDFS checksums on ARM aarch64 architecture**
+
+Add support for aarch64 CRC instructions
+
+
+---
+
+* [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | *Major* | **Rework the changelog and releasenotes**
+
+<!-- markdown -->
+* The release notes now only contains JIRA issues with incompatible changes and actual release notes. The generated format has been changed from HTML to markdown.
+
+* The changelog is now automatically generated from data stored in JIRA rather than manually maintained. The format has been changed from pure text to markdown as well as containing more of the information that was previously stored in the release notes.
+
+* In order to generate the changes file, python must be installed.
+
+* New -Preleasedocs profile added to maven in order to trigger this functionality.
+
+
+---
+
+* [YARN-3365](https://issues.apache.org/jira/browse/YARN-3365) | *Major* | **Add support for using the 'tc' tool via container-executor**
+
+Adding support for using the 'tc' tool in batch mode via container-executor. This is a prerequisite for traffic-shaping functionality that is necessary to support outbound bandwidth as a resource in YARN.
+
+
+---
+
+* [HADOOP-11781](https://issues.apache.org/jira/browse/HADOOP-11781) | *Major* | **fix race conditions and add URL support to smart-apply-patch.sh**
+
+Now auto-downloads patch from issue-id; fixed race conditions; fixed bug affecting some patches.
+
+
+---
+
+* [YARN-3443](https://issues.apache.org/jira/browse/YARN-3443) | *Major* | **Create a 'ResourceHandler' subsystem to ease addition of support for new resource types on the NM**
+
+The current cgroups implementation is closely tied to supporting CPU as a resource . This patch separates out CGroups implementation into a reusable class as well as provides a simple ResourceHandler subsystem that will enable us to add support for new resource types on the NM - e.g Network, Disk etc.
+
+
+---
+
+* [HDFS-6666](https://issues.apache.org/jira/browse/HDFS-6666) | *Minor* | **Abort NameNode and DataNode startup if security is enabled but block access token is not enabled.**
+
+NameNode and DataNode now abort during startup if attempting to run in secure mode, but block access tokens are not enabled by setting configuration property dfs.block.access.token.enable to true in hdfs-site.xml. Previously, this case logged a warning, because this would be an insecure configuration.
+
+
+---
+
+* [YARN-3021](https://issues.apache.org/jira/browse/YARN-3021) | *Major* | **YARN's delegation-token handling disallows certain trust setups to operate properly over DistCp**
+
+ResourceManager renews delegation tokens for applications. This behavior has been changed to renew tokens only if the token's renewer is a non-empty string. MapReduce jobs can instruct ResourceManager to skip renewal of tokens obtained from certain hosts by specifying the hosts with configuration mapreduce.job.hdfs-servers.token-renewal.exclude=\<host1\>,\<host2\>,..,\<hostN\>.
+
+
+---
+
+* [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | *Major* | **rewrite test-patch.sh**
+
+<!-- markdown -->
+* test-patch.sh now has new output that is different than the previous versions
+* test-patch.sh is now pluggable via the test-patch.d directory, with checkstyle and shellcheck tests included
+* JIRA comments now use much more markup to improve readability
+* test-patch.sh now supports either a file name, a URL, or a JIRA issue as input in developer mode
+* If part of the patch testing code is changed, test-patch.sh will now attempt to re-executing itself using the new version.
+* Some logic to try and reduce the amount of unnecessary tests. For example, patches that only modify markdown should not run the Java compilation tests.
+* Plugins for checkstyle, shellcheck, and whitespace now execute as necessary.
+* New test code for mvn site
+* A breakdown of the times needed to execute certain blocks as well as a total runtime is now reported to assist in fixing long running tests and optimize the entire process.
+* Several new options
+ * --resetrepo will put test-patch.sh in destructive mode, similar to a normal Jenkins run
+ * --testlist allows one to provide a comma delimited list of test subsystems to forcibly execute
+ * --modulelist to provide a comma delimited list of module tests to execute in addition to the ones that are automatically detected
+ * --offline mode to attempt to stop connecting to the Internet for certain operations
+* test-patch.sh now defaults to the POSIX equivalents on Solaris and Illumos-based operating systems
+* shelldocs.py may be used to generate test-patch.sh API information
+* FindBugs output is now listed on the JIRA comment
+* lots of general code cleanup, including attempts to remove any local state files to reduce potential race conditions
+* Some logic to determine if a patch is for a given major branch using several strategies as well as a particular git ref (using git+ref as part of the name).
+* Some logic to determine if a patch references a particular JIRA issue.
+* Unit tests are only flagged as necessary with native or Java code, since Hadoop has no framework in place yet for other types of unit tests.
+* test-patch now exits with a failure status if problems arise trying to do git checkouts. Previously the exit code was success.
+
+
+---
+
+* [YARN-3366](https://issues.apache.org/jira/browse/YARN-3366) | *Major* | **Outbound network bandwidth : classify/shape traffic originating from YARN containers**
+
+1) A TrafficController class that provides an implementation for traffic shaping using tc.
+2) A ResourceHandler implementation for OutboundBandwidth as a resource - isolation/enforcement using cgroups and tc.
+
+
+---
+
+* [HADOOP-11861](https://issues.apache.org/jira/browse/HADOOP-11861) | *Major* | **test-patch.sh rewrite addendum patch**
+
+<!-- markdown -->
+* --build-native=false should work now
+* --branch option lets one specify a branch to test against on the command line
+* On certain Jenkins machines, the artifact directory sometimes gets deleted from outside the test-patch script. There is now some code to try to detect, alert, and quick exit if that happens.
+* Various semi-critical output and bug fixes
+
+
+---
+
+* [HADOOP-10597](https://issues.apache.org/jira/browse/HADOOP-10597) | *Major* | **RPC Server signals backoff to clients when all request queues are full**
+
+This change introduces a new configuration key used by RPC server to decide whether to send backoff signal to RPC Client when RPC call queue is full. When the feature is enabled, RPC server will no longer block on the processing of RPC requests when RPC call queue is full. It helps to improve quality of service when the service is under heavy load. The configuration key is in the format of "ipc.#port#.backoff.enable" where #port# is the port number that RPC server listens on. For example, if you want to enable the feature for the RPC server that listens on 8020, set ipc.8020.backoff.enable to true.
+
+
+---
+
+* [HADOOP-11627](https://issues.apache.org/jira/browse/HADOOP-11627) | *Major* | **Remove io.native.lib.available**
+
+io.native.lib.available was removed. Always use native libraries if they exist.
+
+
+---
+
+* [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | *Major* | **Make setting up the build environment easier**
+
+Includes a docker based solution for setting up a build environment with minimal effort.
+
+
+---
+
+* [HDFS-7281](https://issues.apache.org/jira/browse/HDFS-7281) | *Major* | **Missing block is marked as corrupted block**
+
+The patch improves the reporting around missing blocks and corrupted blocks.
+
+1. A block is missing if and only if all DNs of its expected replicas are dead.
+2. A block is corrupted if and only if all its available replicas are corrupted. So if a block has 3 replicas; one of the DN is dead, the other two replicas are corrupted; it will be marked as corrupted.
+3. A new line is added to fsck output to display the corrupt block size per file.
+4. A new line is added to fsck output to display the number of missing blocks in the summary section.
+
+
+---
+
+* [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | *Minor* | **releasedocmaker.py should use today's date instead of unreleased**
+
+Use today instead of 'Unreleased' in releasedocmaker.py when --usetoday is given as an option.
+
+
+---
+
+* [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | *Blocker* | **Non-HA rollback compatibility broken**
+
+Non-HA rollback steps have been changed. Run the rollback command on the namenode (`bin/hdfs namenode -rollback`) before starting cluster with '-rollback' option using (sbin/start-dfs.sh -rollback).
+
+
+---
+
+* [MAPREDUCE-2632](https://issues.apache.org/jira/browse/MAPREDUCE-2632) | *Major* | **Avoid calling the partitioner when the numReduceTasks is 1.**
+
+A partitioner is now only created if there are multiple reducers.
+
+
+---
+
+* [HDFS-8241](https://issues.apache.org/jira/browse/HDFS-8241) | *Minor* | **Remove unused NameNode startup option -finalize**
+
+Remove -finalize option from hdfs namenode command.
+
+
+---
+
+* [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | *Major* | **Allow selectively audit logging ops**
+
+Specific HDFS ops can be selectively excluded from audit logging via 'dfs.namenode.audit.log.debug.cmdlist' configuration.
+
+
+---
+
+* [HDFS-8157](https://issues.apache.org/jira/browse/HDFS-8157) | *Major* | **Writes to RAM DISK reserve locked memory for block files**
+
+This change requires setting the dfs.datanode.max.locked.memory configuration key to use the HDFS Lazy Persist feature. Its value limits the combined off-heap memory for blocks in RAM via caching and lazy persist writes.
+
+
+---
+
+* [HDFS-8332](https://issues.apache.org/jira/browse/HDFS-8332) | *Major* | **DFS client API calls should check filesystem closed**
+
+Users may need special attention for this change while upgrading to this version. Previously user could call some APIs(example: setReplication) wrongly even after closing the fs object. With this change DFS client will not allow any operations to call on closed fs objects. As calling fs operations on closed fs is not right thing to do, users need to correct the usage if any.
+
+
+---
+
+* [HADOOP-11698](https://issues.apache.org/jira/browse/HADOOP-11698) | *Major* | **Remove DistCpV1 and Logalyzer**
+
+Removed DistCpV1 and Logalyzer.
+
+
+---
+
+* [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | *Major* | **RPC Invoker relies on static ClientCache which has synchronized(this) blocks**
+
+The Client#call() methods that are deprecated since 0.23 have been removed.
+
+
+---
+
+* [YARN-3684](https://issues.apache.org/jira/browse/YARN-3684) | *Major* | **Change ContainerExecutor's primary lifecycle methods to use a more extensible mechanism for passing information.**
+
+Modifying key methods in ContainerExecutor to use context objects instead of an argument list. This is more extensible and less brittle.
+
+
+---
+
+* [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | *Major* | **Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HDFS-8135](https://issues.apache.org/jira/browse/HDFS-8135) | *Major* | **Remove the deprecated FSConstants class**
+
+The FSConstants class has been deprecated since 0.23 and it is removed in the release.
+
+
+---
+
+* [MAPREDUCE-6336](https://issues.apache.org/jira/browse/MAPREDUCE-6336) | *Major* | **Enable v2 FileOutputCommitter by default**
+
+mapreduce.fileoutputcommitter.algorithm.version now defaults to 2.
+
+In algorithm version 1:
+
+ 1. commitTask renames directory
+ $joboutput/\_temporary/$appAttemptID/\_temporary/$taskAttemptID/
+ to
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+
+ 2. recoverTask renames
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+ to
+ $joboutput/\_temporary/($appAttemptID + 1)/$taskID/
+
+ 3. commitJob merges every task output file in
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+ to
+ $joboutput/, then it will delete $joboutput/\_temporary/
+ and write $joboutput/\_SUCCESS
+
+commitJob's run time, number of RPC, is O(n) in terms of output files, which is discussed in MAPREDUCE-4815, and can take minutes.
+
+Algorithm version 2 changes the behavior of commitTask, recoverTask, and commitJob.
+
+ 1. commitTask renames all files in
+ $joboutput/\_temporary/$appAttemptID/\_temporary/$taskAttemptID/
+ to $joboutput/
+
+ 2. recoverTask is a nop strictly speaking, but for
+ upgrade from version 1 to version 2 case, it checks if there
+ are any files in
+ $joboutput/\_temporary/($appAttemptID - 1)/$taskID/
+ and renames them to $joboutput/
+
+ 3. commitJob deletes $joboutput/\_temporary and writes
+ $joboutput/\_SUCCESS
+
+Algorithm 2 takes advantage of task parallelism and makes commitJob itself O(1). However, the window of vulnerability for having incomplete output in $jobOutput directory is much larger. Therefore, pipeline logic for consuming job outputs should be built on checking for existence of \_SUCCESS marker.
+
+
+---
+
+* [YARN-2355](https://issues.apache.org/jira/browse/YARN-2355) | *Major* | **MAX\_APP\_ATTEMPTS\_ENV may no longer be a useful env var for a container**
+
+Removed consumption of the MAX\_APP\_ATTEMPTS\_ENV environment variable
+
+
+---
+
+* [HDFS-5033](https://issues.apache.org/jira/browse/HDFS-5033) | *Minor* | **Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source**
+
+"Permission denied" error message when unable to read local file for -put/copyFromLocal
+
+
+---
+
+* [HADOOP-9905](https://issues.apache.org/jira/browse/HADOOP-9905) | *Major* | **remove dependency of zookeeper for hadoop-client**
+
+Zookeeper jar removed from hadoop-client dependency tree.
+
+
+---
+
+* [HDFS-8486](https://issues.apache.org/jira/browse/HDFS-8486) | *Blocker* | **DN startup may cause severe data loss**
+
+<!-- markdown -->
+Public service notice:
+* Every restart of a 2.6.x or 2.7.0 DN incurs a risk of unwanted block deletion.
+* Apply this patch if you are running a pre-2.7.1 release.
+
+
+---
+
+* [HDFS-8270](https://issues.apache.org/jira/browse/HDFS-8270) | *Major* | **create() always retried with hardcoded timeout when file already exists with open lease**
+
+Proxy level retries will not be done on AlreadyBeingCreatedExeption for create() op.
+
+
+---
+
+* [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | *Major* | **The RM should handle the graceful shutdown of the NM.**
+
+The behavior of shutdown a NM could be different (if NM work preserving is not enabled): NM will unregister to RM immediately rather than waiting for timeout to be LOST. A new status of NodeStatus - SHUTDOWN is involved which could affect UI, CLI and ClusterMetrics for node's status.
+
+
+---
+
+* [HDFS-8591](https://issues.apache.org/jira/browse/HDFS-8591) | *Minor* | **Remove support for deprecated configuration key dfs.namenode.decommission.nodes.per.interval**
+
+Related to the decommission enhancements in HDFS-7411, this change removes the deprecated configuration key "dfs.namenode.decommission.nodes.per.interval" which has been subsumed by the configuration key "dfs.namenode.decommission.blocks.per.interval".
+
+
+---
+
+* [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | *Major* | **Allow appending to existing SequenceFiles**
+
+Existing sequence files can be appended.
+
+
+---
+
+* [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | *Minor* | **Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings**
+
+Add a new option "properties" to the "dfsadmin -reconfig" command to get a list of reconfigurable properties.
+
+
+---
+
+* [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | *Major* | **Use slf4j instead of common-logging in hdfs-client**
+
+Users may need special attention for this change while upgrading to this version. Previously hdfs client was using commons-logging as the logging framework. With this change it will use slf4j framework. For more details about slf4j, please see: http://www.slf4j.org/manual.html. Also, org.apache.hadoop.hdfs.protocol.CachePoolInfo#LOG public static member variable has been removed as it is not used anywhere. Users need to correct their code if any one has a reference to this variable. One can retrieve the named logger via the logging framework of their choice directly like, org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(org.apache.hadoop.hdfs.protocol.CachePoolInfo.class);
+
+
+---
+
+* [HDFS-6440](https://issues.apache.org/jira/browse/HDFS-6440) | *Major* | **Support more than 2 NameNodes**
+
+This feature adds support for running additional standby NameNodes, which provides additional fault-tolerance. It is designed for a total of 3-5 NameNodes.
+
+
+---
+
+* [YARN-3823](https://issues.apache.org/jira/browse/YARN-3823) | *Minor* | **Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property**
+
+Default value for 'yarn.scheduler.maximum-allocation-vcores' changed from 32 to 4.
+
+
+---
+
+* [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | *Minor* | **Add SFTP FileSystem**
+
+Added SFTP filesystem by using the JSch library.
+
+
+---
+
+* [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | *Major* | **Document missing properties in yarn-default.xml**
+
+Documented missing properties and added the regression test to verify that there are no missing properties in yarn-default.xml.
+
+
+---
+
+* [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | *Minor* | **Fix typo in JobHistoryEventHandler**
+
+There is a typo in the event string "WORKFLOW\_ID" (as "WORKLFOW\_ID"). The branch-2 change will publish both event strings for compatibility with consumers, but the misspelled metric will be removed in trunk.
+
+
+---
+
+* [HDFS-7582](https://issues.apache.org/jira/browse/HDFS-7582) | *Major* | **Enforce maximum number of ACL entries separately per access and default.**
+
+Limit on Maximum number of ACL entries(32) will be enforced separately on access and default ACLs. So in total, max. 64 ACL entries can be present in a ACL spec.
+
+
+---
+
+* [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | *Major* | **Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3**
+
+The Maven dependency on aws-sdk has been changed to aws-sdk-s3 and the version bumped. Applications depending on transitive dependencies pulled in by aws-sdk and not aws-sdk-s3 might not work.
+
+
+---
+
+* [HDFS-8818](https://issues.apache.org/jira/browse/HDFS-8818) | *Major* | **Allow Balancer to run faster**
+
+Add a new conf "dfs.balancer.max-size-to-move" so that Balancer.MAX\_SIZE\_TO\_MOVE becomes configurable.
+
+
+---
+
+* [HDFS-8895](https://issues.apache.org/jira/browse/HDFS-8895) | *Major* | **Remove deprecated BlockStorageLocation APIs**
+
+This removes the deprecated DistributedFileSystem#getFileBlockStorageLocations API used for getting VolumeIds of block replicas. Applications interested in the volume of a replica can instead consult BlockLocation#getStorageIds to obtain equivalent information.
+
+
+---
+
+* [HADOOP-12352](https://issues.apache.org/jira/browse/HADOOP-12352) | *Trivial* | **Delay in checkpointing Trash can leave trash for 2 intervals before deleting**
+
+Fixes an Trash related issue wherein a delay in the periodic checkpointing of one user's directory causes the subsequent user directory checkpoints to carry a newer timestamp, thereby delaying their eventual deletion.
+
+
+---
+
+* [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | *Major* | **Compact XAttrs to optimize memory footprint.**
+
+The config key "dfs.namenode.fs-limits.max-xattr-size" can no longer be set to a value of 0 (previously used to indicate unlimited) or a value greater than 32KB. This is a constraint on xattr size similar to many local filesystems.
+
+
+---
+
+* [HDFS-8890](https://issues.apache.org/jira/browse/HDFS-8890) | *Major* | **Allow admin to specify which blockpools the balancer should run on**
+
+Adds a new blockpools flag to the balancer. This allows admins to specify which blockpools the balancer will run on.
+Usage:
+-blockpools \<comma-separated list of blockpool ids\>
+The balancer will only run on blockpools included in this list.
+
+
+---
+
+* [HDFS-8981](https://issues.apache.org/jira/browse/HDFS-8981) | *Minor* | **Adding revision to data node jmx getVersion() method**
+
+getSoftwareVersion method would replace original getVersion method, which returns the version string.
+
+The new getVersion method would return both version string and revision string.
+
+
+---
+
+* [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | *Major* | **Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HADOOP-12384](https://issues.apache.org/jira/browse/HADOOP-12384) | *Major* | **Add "-direct" flag option for fs copy so that user can choose not to create ".\_COPYING\_" file**
+
+An option '-d' added for all command-line copy commands to skip intermediate '.COPYING' file creation.
+
+
+---
+
+* [HDFS-8929](https://issues.apache.org/jira/browse/HDFS-8929) | *Major* | **Add a metric to expose the timestamp of the last journal**
+
+Exposed a metric 'LastJournalTimestamp' for JournalNode
+
+
+---
+
+* [HDFS-7116](https://issues.apache.org/jira/browse/HDFS-7116) | *Major* | **Add a command to get the balancer bandwidth**
+
+Exposed command "-getBalancerBandwidth" in dfsadmin to get the bandwidth of balancer.
+
+
+---
+
+* [HDFS-8829](https://issues.apache.org/jira/browse/HDFS-8829) | *Major* | **Make SO\_RCVBUF and SO\_SNDBUF size configurable for DataTransferProtocol sockets and allow configuring auto-tuning**
+
+HDFS-8829 introduces two new configuration settings: dfs.datanode.transfer.socket.send.buffer.size and dfs.datanode.transfer.socket.recv.buffer.size. These settings can be used to control the socket send buffer and receive buffer sizes respectively on the DataNode for client-DataNode and DataNode-DataNode connections. The default values of both settings are 128KB for backwards compatibility. For optimum performance it is recommended to set these values to zero to enable the OS networking stack to auto-tune buffer sizes.
+
+
+---
+
+* [YARN-313](https://issues.apache.org/jira/browse/YARN-313) | *Critical* | **Add Admin API for supporting node resource configuration in command line**
+
+After this patch, the feature to support NM resource dynamically configuration is completed, so that user can configure NM with new resource without bring NM down or decommissioned.
+Two CLIs are provided to support update resources on individual node or a batch of nodes:
+1. Update resource on single node: yarn rmadmin -updateNodeResource [NodeID] [MemSize] [vCores]
+2. Update resource on a batch of nodes: yarn rmadmin -refreshNodesResources, that reflect nodes' resource configuration defined in dynamic-resources.xml which is loaded by RM dynamically (like capacity-scheduler.xml or fair-scheduler.xml).
+The first version of configuration format is:
+\<configuration\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.nodes\</name\>
+ \<value\>h1:1234\</value\>
+ \</property\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.h1:1234.vcores\</name\>
+ \<value\>16\</value\>
+ \</property\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.h1:1234.memory\</name\>
+ \<value\>1024\</value\>
+ \</property\>
+\</configuration\>
+
+
+---
+
+* [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | *Major* | **Trash messages should be handled by Logger instead of being delivered on System.out**
+
+Now trash message is not printed to System.out. It is handled by Logger instead.
+
+
+---
+
+* [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | *Minor* | **Create StatsD metrics2 sink**
+
+Added StatsD metrics2 sink
+
+
+---
+
+* [HDFS-9013](https://issues.apache.org/jira/browse/HDFS-9013) | *Major* | **Deprecate NameNodeMXBean#getNNStarted in branch2 and remove from trunk**
+
+NameNodeMXBean#getNNStarted() metric is deprecated in branch-2 and removed from trunk.
+
+
+---
+
+* [HADOOP-12437](https://issues.apache.org/jira/browse/HADOOP-12437) | *Major* | **Allow SecurityUtil to lookup alternate hostnames**
+
+HADOOP-12437 introduces two new configuration settings: hadoop.security.dns.interface and hadoop.security.dns.nameserver. These settings can be used to control how Hadoop service instances look up their own hostname and may be required in some multi-homed environments where hosts are configured with multiple hostnames in DNS or hosts files. They supersede the existing settings dfs.datanode.dns.interface and dfs.datanode.dns.nameserver.
+
+
+---
+
+* [HADOOP-12446](https://issues.apache.org/jira/browse/HADOOP-12446) | *Major* | **Undeprecate createNonRecursive()**
+
+FileSystem#createNonRecursive() is undeprecated.
+
+
+---
+
+* [HDFS-8696](https://issues.apache.org/jira/browse/HDFS-8696) | *Major* | **Make the lower and higher watermark in the DN Netty server configurable**
+
+Introduced two new configuration dfs.webhdfs.netty.low.watermark and dfs.webhdfs.netty.high.watermark to enable tuning the size of the buffers of the Netty server inside Datanodes.
+
+
+---
+
+* [HDFS-7285](https://issues.apache.org/jira/browse/HDFS-7285) | *Major* | **Erasure Coding Support inside HDFS**
+
+<!-- markdown -->
+HDFS now provides native support for erasure coding (EC) to store data more efficiently. Each individual directory can be configured with an EC policy with command `hdfs erasurecode -setPolicy`. When a file is created, it will inherit the EC policy from its nearest ancestor directory to determine how its blocks are stored. Compared to 3-way replication, the default EC policy saves 50% of storage space while also tolerating more storage failures.
+
+To support small files, the currently phase of HDFS-EC stores blocks in _striped_ layout, where a logical file block is divided into small units (64KB by default) and distributed to a set of DataNodes. This enables parallel I/O but also decreases data locality. Therefore, the cluster environment and I/O workloads should be considered before configuring EC policies.
+
+
+---
+
+* [HDFS-9085](https://issues.apache.org/jira/browse/HDFS-9085) | *Trivial* | **Show renewer information in DelegationTokenIdentifier#toString**
+
+The output of the "hdfs fetchdt --print" command now includes the token renewer appended to the end of the existing token information. This change may be incompatible with tools that parse the output of the command.
+
+
+---
+
+* [HADOOP-12493](https://issues.apache.org/jira/browse/HADOOP-12493) | *Major* | **bash unit tests are failing**
+
+In the extremely rare event that HADOOP\_USER\_IDENT and USER environment variables are not defined, we now fall back to use 'hadoop' as the identification string.
+
+
+---
+
+* [HADOOP-12495](https://issues.apache.org/jira/browse/HADOOP-12495) | *Major* | **Fix posix\_spawn error on OS X**
+
+When Hadoop JVMs create other processes on OS X, it will always use posix\_spawn.
+
+
+---
+
+* [HDFS-9070](https://issues.apache.org/jira/browse/HDFS-9070) | *Major* | **Allow fsck display pending replica location information for being-written blocks**
+
+The output of fsck command for being written hdfs files had been changed. When using fsck against being written hdfs files with {{-openforwrite}} and {{-files -blocks -locations}}, the fsck output will include the being written block for replication files or being written block group for erasure code files.
+
+
+---
+
+* [HDFS-9278](https://issues.apache.org/jira/browse/HDFS-9278) | *Trivial* | **Fix preferredBlockSize typo in OIV XML output**
+
+The preferred block size XML element has been corrected from "\\<perferredBlockSize\>" to "\\<preferredBlockSize\>".
+
+
+---
+
+* [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | *Major* | **Logging HDFS operation's caller context into audit logs**
+
+The feature needs to enabled by setting "hadoop.caller.context.enabled" to true. When the feature is used, additional fields are written into namenode audit log records.
+
+
+---
+
+* [HDFS-9259](https://issues.apache.org/jira/browse/HDFS-9259) | *Major* | **Make SO\_SNDBUF size configurable at DFSClient side for hdfs write scenario**
+
+Introduces a new configuration setting dfs.client.socket.send.buffer.size to control the socket send buffer size for writes. Setting it to zero enables TCP auto-tuning on systems that support it.
+
+
+---
+
+* [HDFS-9311](https://issues.apache.org/jira/browse/HDFS-9311) | *Major* | **Support optional offload of NameNode HA service health checks to a separate RPC server.**
+
+There is now support for offloading HA health check RPC activity to a separate RPC server endpoint running within the NameNode process. This may improve reliability of HA health checks and prevent spurious failovers in highly overloaded conditions. For more details, please refer to the hdfs-default.xml documentation for properties dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host and dfs.namenode.lifeline.handler.count.
+
+
+---
+
+* [HDFS-6200](https://issues.apache.org/jira/browse/HDFS-6200) | *Major* | **Create a separate jar for hdfs-client**
+
+Projects that access HDFS can depend on the hadoop-hdfs-client module instead of the hadoop-hdfs module to avoid pulling in unnecessary dependency.
+
+
+---
+
+* [HADOOP-10787](https://issues.apache.org/jira/browse/HADOOP-10787) | *Blocker* | **Rename/remove non-HADOOP\_\*, etc from the shell scripts**
+
+<!-- markdown -->
+The following shell environment variables have been deprecated:
+
+| Old | New |
+|:---- |:---- |
+| DEFAULT\_LIBEXEC\_DIR | HADOOP\_DEFAULT\_LIBEXEC\_DIR |
+| SLAVE\_NAMES | HADOOP\_SLAVE\_NAMES |
+| TOOL\_PATH | HADOOP\_TOOLS\_PATH |
+
+In addition:
+
+* DEFAULT\_LIBEXEC\_DIR will NOT be automatically transitioned to HADOOP\_DEFAULT\_LIBEXEC\_DIR and will require changes to any scripts setting that value. A warning will be printed to the screen if DEFAULT\_LIBEXEC\_DIR has been configured.
+* HADOOP\_TOOLS\_PATH is now properly handled as a multi-valued, Java classpath-style variable. Prior, multiple values assigned to TOOL\_PATH would not work a predictable manner.
+
+
+---
+
+* [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | *Major* | **allow/disallow snapshots via webhdfs**
+
+Snapshots can be allowed/disallowed on a directory via WebHdfs from users with superuser privilege.
+
+
+---
+
+* [MAPREDUCE-5485](https://issues.apache.org/jira/browse/MAPREDUCE-5485) | *Critical* | **Allow repeating job commit by extending OutputCommitter API**
+
+Previously, the MR job will get failed if AM get restarted for some reason (like node failure, etc.) during its doing commit job no matter if AM attempts reach to the maximum attempts.
+In this improvement, we add a new API isCommitJobRepeatable() to OutputCommitter interface which to indicate if job's committer can do commitJob again if previous commit work is interrupted by NM/AM failures, etc. The instance of OutputCommitter, which support repeatable job commit (like FileOutputCommitter in algorithm 2), can allow AM to continue the commitJob() after AM restart as a new attempt.
+
+
+---
+
+* [HADOOP-12294](https://issues.apache.org/jira/browse/HADOOP-12294) | *Major* | **Throw an Exception when fs.permissions.umask-mode is misconfigured**
+
+The support of the deprecated dfs.umask key is removed in Hadoop 3.0.
+
+
+---
+
+* [HADOOP-10465](https://issues.apache.org/jira/browse/HADOOP-10465) | *Minor* | **Fix use of generics within SortedMapWritable**
+
+SortedMapWritable has changed to SortedMapWritable\<K extends WritableComparable\<? super K\>\>. That way user can declare the class by such as SortedMapWritable\<Text\>.
+
+
+---
+
+* [HADOOP-12313](https://issues.apache.org/jira/browse/HADOOP-12313) | *Critical* | **NPE in JvmPauseMonitor when calling stop() before start()**
+
+Allow stop() before start() completed in JvmPauseMonitor
+
+
+---
+
+* [HDFS-5165](https://issues.apache.org/jira/browse/HDFS-5165) | *Minor* | **Remove the TotalFiles metrics**
+
+Now TotalFiles metric is removed from FSNameSystem. Use FilesTotal instead.
+
+
+---
+
+* [HDFS-8335](https://issues.apache.org/jira/browse/HDFS-8335) | *Major* | **FSNamesystem should construct FSPermissionChecker only if permission is enabled**
+
+Only check permissions when permissions enabled in FSDirStatAndListingOp.getFileInfo() and getListingInt()
+
+
+---
+
+* [HDFS-8831](https://issues.apache.org/jira/browse/HDFS-8831) | *Major* | **Trash Support for deletion in HDFS encryption zone**
+
+Add Trash support for deleting files within encryption zones. Deleted files will remain encrypted and they will be moved to a \u201c.Trash\u201d subdirectory under the root of the encryption zone, prefixed by $USER/current. Checkpoint and expunge continue to work like the existing Trash.
+
+
+---
+
+* [HDFS-9214](https://issues.apache.org/jira/browse/HDFS-9214) | *Major* | **Support reconfiguring dfs.datanode.balance.max.concurrent.moves without DN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> start
+3. repeat step 2 until all DNs are reconfigured
+4. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> status
+5. to query a list reconfigurable properties on DN, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | *Major* | **We should have a config to indicate the Timeline Service version**
+
+Add a new configuration "yarn.timeline-service.version" to indicate what is the current version of the running timeline service. For example, if "yarn.timeline-service.version" is 1.5, and "yarn.timeline-service.enabled" is true, it means the cluster will and should bring up the timeline service v.1.5. On the client side, if the client uses the same version of timeline service, it should succeed. If the client chooses to use a smaller version in spite of this, then depending on how robust the compatibility story is between versions, the results may vary.
+
+
+---
+
+* [YARN-4207](https://issues.apache.org/jira/browse/YARN-4207) | *Major* | **Add a non-judgemental YARN app completion status**
+
+Adds the ENDED attribute to o.a.h.yarn.api.records.FinalApplicationStatus
+
+
+---
+
+* [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | *Minor* | **Add a option to skip newline on empty files with getMerge -nl**
+
+Added -skip-empty-file option to hadoop fs -getmerge command. With the option, delimiter (LF) is not printed for empty files even if -nl option is used.
+
+
+---
+
+* [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | *Major* | **Retire libwebhdfs**
+
+libwebhdfs has been retired in 2.8.0 due to the lack of maintenance.
+
+
+---
+
+* [HADOOP-11262](https://issues.apache.org/jira/browse/HADOOP-11262) | *Major* | **Enable YARN to use S3A**
+
+S3A has been made accessible through the FileContext API.
+
+
+---
+
+* [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | *Major* | **Adding Append API support for WASB**
+
+The Azure Blob Storage file system (WASB) now includes optional support for use of the append API by a single writer on a path. Please note that the implementation differs from the semantics of HDFS append. HDFS append internally guarantees that only a single writer may append to a path at a given time. WASB does not enforce this guarantee internally. Instead, the application must enforce access by a single writer, such as by running single-threaded or relying on some external locking mechanism to coordinate concurrent processes. Refer to the Azure Blob Storage documentation page for more details on enabling append in configuration.
+
+
+---
+
+* [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | *Major* | **Replace dev-support with wrappers to Yetus**
+
+<!-- markdown -->
+
+* Major portions of dev-support have been replaced with wrappers to Apache Yetus:
+ * releasedocmaker.py is now dev-support/bin/releasedocmaker
+ * shelldocs.py is now dev-support/bin/shelldocs
+ * smart-apply-patch.sh is now dev-support/bin/smart-apply-patch
+ * test-patch.sh is now dev-support/bin/test-patch
+* See the dev-support/README.md file for more details on how to control the wrappers to various degrees.
+
+
+---
+
+* [HDFS-9525](https://issues.apache.org/jira/browse/HDFS-9525) | *Blocker* | **hadoop utilities need to support provided delegation tokens**
+
+If hadoop.token.files property is defined and configured to one or more comma-delimited delegation token files, Hadoop will use those token files to connect to the services as named in the token.
+
+
+---
+
+* [HDFS-9503](https://issues.apache.org/jira/browse/HDFS-9503) | *Major* | **Replace -namenode option with -fs for NNThroughputBenchmark**
+
+The patch replaces -namenode option with -fs for specifying the remote name node against which the benchmark is running. Before this patch, if '-namenode' was not given, the benchmark would run in standalone mode, ignoring the 'fs.defaultFS' in config file even if it's remote. With this patch, the benchmark, as other tools, will rely on the 'fs.defaultFS' config, which is overridable by -fs command option, to run standalone mode or remote mode.
+
+
+---
+
+* [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | *Minor* | **Add Entry point for Kerberos health check**
+
+Hadoop now includes a shell command named KDiag that helps with diagnosis of Kerberos misconfiguration problems. Please refer to the Secure Mode documentation for full details on usage of the command.
+
+
+---
+
+* [HADOOP-12805](https://issues.apache.org/jira/browse/HADOOP-12805) | *Major* | **Annotate CanUnbuffer with @InterfaceAudience.Public**
+
+Made CanBuffer interface public for use in client applications.
+
+
+---
+
+* [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | *Major* | **Read s3a creds from a Credential Provider**
+
+The S3A Hadoop-compatible file system now support reading its S3 credentials from the Hadoop Credential Provider API in addition to XML configuration files.
+
+
+---
+
+* [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | *Major* | **Integrate CSRF prevention filter in WebHDFS.**
+
+WebHDFS now supports options to enforce cross-site request forgery (CSRF) prevention for HTTP requests to both the NameNode and the DataNode. Please refer to the updated WebHDFS documentation for a description of this feature and further details on how to configure it.
+
+
+---
+
+* [HADOOP-12794](https://issues.apache.org/jira/browse/HADOOP-12794) | *Major* | **Support additional compression levels for GzipCodec**
+
+Added New compression levels for GzipCodec that can be set in zlib.compress.level
+
+
+---
+
+* [MAPREDUCE-6613](https://issues.apache.org/jira/browse/MAPREDUCE-6613) | *Minor* | **Change mapreduce.jobhistory.jhist.format default from json to binary**
+
+Default of 'mapreduce.jobhistory.jhist.format' property changed from 'json' to 'binary'. Creates smaller, binary Avro .jhist files for faster JHS performance.
+
+
+---
+
+* [HDFS-9425](https://issues.apache.org/jira/browse/HDFS-9425) | *Major* | **Expose number of blocks per volume as a metric**
+
+Number of blocks per volume is made available as a metric.
+
+
+---
+
+* [HADOOP-12668](https://issues.apache.org/jira/browse/HADOOP-12668) | *Critical* | **Support excluding weak Ciphers in HttpServer2 through ssl-server.conf**
+
+The Code Changes include following:
+- Modified DFSUtil.java in Apache HDFS project for supplying new parameter ssl.server.exclude.cipher.list
+- Modified HttpServer2.java in Apache Hadoop-common project to work with new parameter and exclude ciphers using jetty setExcludeCihers method.
+- Modfied associated test classes to owrk with existing code and also cover the newfunctionality in junit
+
+
+---
+
+* [HADOOP-12555](https://issues.apache.org/jira/browse/HADOOP-12555) | *Minor* | **WASB to read credentials from a credential provider**
+
+The hadoop-azure file system now supports configuration of the Azure Storage account credentials using the standard Hadoop Credential Provider API. For details, please refer to the documentation on hadoop-azure and the Credential Provider API.
+
+
+---
+
+* [HDFS-9395](https://issues.apache.org/jira/browse/HDFS-9395) | *Major* | **Make HDFS audit logging consistant**
+
+<!-- markdown -->
+
+Audit logs will now only be generated in the following two cases:
+* When an operation results in an `AccessControlException`
+* When an operation is successful
+
+Notably, this means audit log events will not be generated for exceptions besides AccessControlException.
+
+
+---
+
+* [MAPREDUCE-6622](https://issues.apache.org/jira/browse/MAPREDUCE-6622) | *Critical* | **Add capability to set JHS job cache to a task-based limit**
+
+Two recommendations for the mapreduce.jobhistory.loadedtasks.cache.size property:
+1) For every 100k of cache size, set the heap size of the Job History Server to 1.2GB. For example, mapreduce.jobhistory.loadedtasks.cache.size=500000, heap size=6GB.
+2) Make sure that the cache size is larger than the number of tasks required for the largest job run on the cluster. It might be a good idea to set the value slightly higher (say, 20%) in order to allow for job size growth.
+
+
+---
+
+* [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | *Minor* | **Fix undeclared/unused dependency to httpclient**
+
+Dependency on commons-httpclient::commons-httpclient was removed from hadoop-common. Downstream projects using commons-httpclient transitively provided by hadoop-common need to add explicit dependency to their pom. Since commons-httpclient is EOL, it is recommended to migrate to org.apache.httpcomponents:httpclient which is the successor.
+
+
+---
+
+* [HADOOP-12850](https://issues.apache.org/jira/browse/HADOOP-12850) | *Major* | **pull shell code out of hadoop-dist**
+
+This change contains the content of HADOOP-10115 which is an incompatible change.
+
+
+---
+
+* [HDFS-8791](https://issues.apache.org/jira/browse/HDFS-8791) | *Blocker* | **block ID-based DN storage layout can be very slow for datanode on ext4**
+
+HDFS-8791 introduces a new datanode layout format. This layout is identical to the previous block id based layout except it has a smaller 32x32 sub-directory structure in each data storage. On startup, the datanode will automatically upgrade it's storages to this new layout. Currently, datanode layout changes support rolling upgrades, on the other hand downgrading is not supported between datanode layout changes and a rollback would be required.
+
+
+---
+
+* [HDFS-9887](https://issues.apache.org/jira/browse/HDFS-9887) | *Major* | **WebHdfs socket timeouts should be configurable**
+
+Added new configuration options: dfs.webhdfs.socket.connect-timeout and dfs.webhdfs.socket.read-timeout both defaulting to 60s.
+
+
+---
+
+* [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | *Major* | **Remove all of the CHANGES.txt files**
+
+With the introduction of the markdown-formatted and automatically built changes file, the CHANGES.txt files have been eliminated.
+
+
+---
+
+* [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | *Major* | **DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness**
+
+This release adds a new feature called the DataNode Lifeline Protocol. If configured, then DataNodes can report that they are still alive to the NameNode via a fallback protocol, separate from the existing heartbeat messages. This can prevent the NameNode from incorrectly marking DataNodes as stale or dead in highly overloaded clusters where heartbeat processing is suffering delays. For more information, please refer to the hdfs-default.xml documentation for several new configuration properties: dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host, dfs.datanode.lifeline.interval.seconds, dfs.namenode.lifeline.handler.ratio and dfs.namenode.lifeline.handler.count.
+
+
+---
+
+* [YARN-4762](https://issues.apache.org/jira/browse/YARN-4762) | *Blocker* | **NMs failing on DelegatingLinuxContainerRuntime init with LCE on**
+
+Fixed CgroupHandler's creation and usage to avoid NodeManagers crashing when LinuxContainerExecutor is enabled.
+
+
+---
+
+* [HDFS-1477](https://issues.apache.org/jira/browse/HDFS-1477) | *Major* | **Support reconfiguring dfs.heartbeat.interval and dfs.namenode.heartbeat.recheck-interval without NN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> start
+3. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> status
+4. to query a list reconfigurable properties on NN, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | *Major* | **inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [YARN-4732](https://issues.apache.org/jira/browse/YARN-4732) | *Trivial* | **\*ProcessTree classes have too many whitespace issues**
+
+
+
+
+---
+
+* [HADOOP-12857](https://issues.apache.org/jira/browse/HADOOP-12857) | *Major* | **Rework hadoop-tools**
+
+<!-- markdown -->
+* Turning on optional things from the tools directory such as S3 support can now be done in hadoop-env.sh with the HADOOP\_OPTIONAL\_TOOLS environment variable without impacting the various user-facing CLASSPATH variables.
+* The tools directory is no longer pulled in blindly for any utilities that pull it in.
+* TOOL\_PATH / HADOOP\_TOOLS\_PATH has been broken apart and replaced with HADOOP\_TOOLS\_HOME, HADOOP\_TOOLS\_DIR and HADOOP\_TOOLS\_LIB\_JARS\_DIR to be consistent with the rest of Hadoop.
+
+
+---
+
+* [HDFS-9694](https://issues.apache.org/jira/browse/HDFS-9694) | *Major* | **Make existing DFSClient#getFileChecksum() work for striped blocks**
+
+Makes the getFileChecksum API works with striped layout EC files. Checksum computation done by block level in the distributed fashion. The current API does not support to compare the checksum generated with normal file and the checksum generated for the same file but in striped layout.
+
+
+---
+
+* [HDFS-9640](https://issues.apache.org/jira/browse/HDFS-9640) | *Major* | **Remove hsftp from DistCp in trunk**
+
+DistCp in Hadoop 3.0 no longer supports -mapredSSLConf option. Use global ssl-client.xml configuration file for swebhdfs file systems instead.
+
+
+---
+
+* [HDFS-9349](https://issues.apache.org/jira/browse/HDFS-9349) | *Major* | **Support reconfiguring fs.protected.directories without NN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> start
+3. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> status
+4. to query a list reconfigurable properties on NN, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [HADOOP-11393](https://issues.apache.org/jira/browse/HADOOP-11393) | *Major* | **Revert HADOOP\_PREFIX, go back to HADOOP\_HOME**
+
+On Unix platforms, HADOOP\_PREFIX has been deprecated in favor of returning to HADOOP\_HOME as in prior Apache Hadoop releases.
+
+
+---
+
+* [HADOOP-12967](https://issues.apache.org/jira/browse/HADOOP-12967) | *Major* | **Remove FileUtil#copyMerge**
+
+Removed FileUtil.copyMerge.
+
+
+---
+
+* [MAPREDUCE-6670](https://issues.apache.org/jira/browse/MAPREDUCE-6670) | *Minor* | **TestJobListCache#testEviction sometimes fails on Windows with timeout**
+
+Backport the fix to 2.7 and 2.8
+
+
+---
+
+* [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | *Major* | **Datanode command for evicting writers**
+
+This new dfsadmin command, evictWriters, stops active block writing activities on a data node. The affected writes will continue without the node after a write pipeline recovery. This is useful when data node decommissioning is blocked by slow writers. If issued against a non-decommissioing data node, all current writers will be stopped, but new write requests will continue to be served.
+
+
+---
+
+* [HADOOP-12963](https://issues.apache.org/jira/browse/HADOOP-12963) | *Minor* | **Allow using path style addressing for accessing the s3 endpoint**
+
+Add new flag to allow supporting path style addressing for s3a
+
+
+---
+
+* [HADOOP-12811](https://issues.apache.org/jira/browse/HADOOP-12811) | *Critical* | **Change kms server port number which conflicts with HMaster port number**
+
+The default port for KMS service is now 9600. This is to avoid conflicts on the previous port 16000, which is also used by HMaster as the default port.
+
+
+---
+
+* [YARN-4784](https://issues.apache.org/jira/browse/YARN-4784) | *Major* | **Fairscheduler: defaultQueueSchedulingPolicy should not accept FIFO**
+
+Clusters cannot use FIFO policy as the defaultQueueSchedulingPolicy. Clusters with a single level of queues will have to explicitly set the policy to FIFO if that is desired.
+
+
+---
+
+* [HDFS-9427](https://issues.apache.org/jira/browse/HDFS-9427) | *Critical* | **HDFS should not default to ephemeral ports**
+
+The patch updates the HDFS default HTTP/RPC ports to non-ephemeral ports. The changes are listed below:
+Namenode ports: 50470 --\> 9871, 50070 --\> 9870, 8020 --\> 9820
+Secondary NN ports: 50091 --\> 9869, 50090 --\> 9868
+Datanode ports: 50020 --\> 9867, 50010 --\> 9866, 50475 --\> 9865, 50075 --\> 9864
+
+
+---
+
+* [HDFS-3702](https://issues.apache.org/jira/browse/HDFS-3702) | *Minor* | **Add an option for NOT writing the blocks locally if there is a datanode on the same box as the client**
+
+This patch will attempt to allocate all replicas to remote DataNodes, by adding local DataNode to the excluded DataNodes. If no sufficient replicas can be obtained, it will fall back to default block placement policy, which writes one replica to local DataNode.
+
+
+---
+
+* [HADOOP-12563](https://issues.apache.org/jira/browse/HADOOP-12563) | *Major* | **Updated utility to create/modify token files**
+
+This feature introduces a new command called "hadoop dtutil" which lets users request and download delegation tokens with certain attributes.
+
+
+---
+
+* [HADOOP-13045](https://issues.apache.org/jira/browse/HADOOP-13045) | *Major* | **hadoop\_add\_classpath is not working in .hadooprc**
+
+<!-- markdown -->
+With this change, the `.hadooprc` file is now processed after Apache Hadoop has been fully bootstrapped. This allows for usage of the Apache Hadoop Shell API. A new file, `.hadoop-env`, now provides the ability for end users to override `hadoop-env.sh`.
+
+
+---
+
+* [MAPREDUCE-6526](https://issues.apache.org/jira/browse/MAPREDUCE-6526) | *Blocker* | **Remove usage of metrics v1 from hadoop-mapreduce**
+
+LocalJobRunnerMetrics and ShuffleClientMetrics were updated to use Hadoop Metrics V2 framework.
+
+
+---
+
+* [HDFS-9902](https://issues.apache.org/jira/browse/HDFS-9902) | *Major* | **Support different values of dfs.datanode.du.reserved per storage type**
+
+Reserved space can be configured independently for different storage types for clusters with heterogeneous storage. The 'dfs.datanode.du.reserved' property name can be suffixed with a storage types (i.e. one of ssd, disk, archival or ram\_disk). e.g. reserved space for RAM\_DISK storage can be configured using the property 'dfs.datanode.du.reserved.ram\_disk'. If specific storage type reservation is not configured then the value specified by 'dfs.datanode.du.reserved' will be used for all volumes.
+
+
+---
+
+* [HADOOP-12504](https://issues.apache.org/jira/browse/HADOOP-12504) | *Blocker* | **Remove metrics v1**
+
+<!-- markdown -->
+* org.apache.hadoop.metrics package was removed. Use org.apache.hadoop.metrics2 package instead.
+* "/metrics" endpoint was removed. Use "/jmx" instead to see the metrics.
+
+
+---
+
+* [HDFS-10324](https://issues.apache.org/jira/browse/HDFS-10324) | *Major* | **Trash directory in an encryption zone should be pre-created with correct permissions**
+
+HDFS will create a ".Trash" subdirectory when creating a new encryption zone to support soft delete for files deleted within the encryption zone. A new "crypto -provisionTrash" command has been introduced to provision trash directories for encryption zones created with Apache Hadoop minor releases prior to 2.8.0.
+
+
+---
+
+* [HDFS-10337](https://issues.apache.org/jira/browse/HDFS-10337) | *Minor* | **OfflineEditsViewer stats option should print 0 instead of null for the count of operations**
+
+The output of "hdfs oev -p stats" has changed. The option prints 0 instead of null for the count of the operations that have never been executed.
+
+
+---
+
+* [HADOOP-10694](https://issues.apache.org/jira/browse/HADOOP-10694) | *Major* | **Remove synchronized input streams from Writable deserialization**
+
+Remove invisible synchronization primitives from DataInputBuffer
+
+
+---
+
+* [HADOOP-13122](https://issues.apache.org/jira/browse/HADOOP-13122) | *Minor* | **Customize User-Agent header sent in HTTP requests by S3A.**
+
+S3A now includes the current Hadoop version in the User-Agent string passed through the AWS SDK to the S3 service. Users also may include optional additional information to identify their application. See the documentation of configuration property fs.s3a.user.agent.prefix for further details.
+
+
+---
+
+* [HADOOP-11858](https://issues.apache.org/jira/browse/HADOOP-11858) | *Blocker* | **[JDK8] Set minimum version of Hadoop 3 to JDK 8**
+
+The minimum required JDK version for Hadoop has been increased from JDK7 to JDK8.
+
+
+---
+
+* [HADOOP-12930](https://issues.apache.org/jira/browse/HADOOP-12930) | *Critical* | **[Umbrella] Dynamic subcommands for hadoop shell scripts**
+
+It is now possible to add or modify the behavior of existing subcommands in the hadoop, hdfs, mapred, and yarn scripts. See the Unix Shell Guide for more information.
+
+
+---
+
+* [HADOOP-12782](https://issues.apache.org/jira/browse/HADOOP-12782) | *Major* | **Faster LDAP group name resolution with ActiveDirectory**
+
+If the user object returned by LDAP server has the user's group object DN (supported by Active Directory), Hadoop can reduce LDAP group mapping latency by setting hadoop.security.group.mapping.ldap.search.attr.memberof to memberOf.
+
+
+---
+
+* [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | *Major* | **S3A: Add ability to plug in any AWSCredentialsProvider**
+
+Users can integrate a custom credential provider with S3A. See documentation of configuration property fs.s3a.aws.credentials.provider for further details.
+
+
+---
+
+* [MAPREDUCE-6607](https://issues.apache.org/jira/browse/MAPREDUCE-6607) | *Minor* | **Enable regex pattern matching when mapreduce.task.files.preserve.filepattern is set**
+
+Before this fix, the files in .staging directory are always preserved when mapreduce.task.files.preserve.filepattern is set. After this fix, the files in .staging directory are preserved if the name of the directory matches the regex pattern specified by mapreduce.task.files.preserve.filepattern.
+
+
+---
+
+* [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | *Major* | **FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on**
+
+Introducing a new configuration "yarn.scheduler.fair.dynamic.max.assign" to dynamically determine the resources to assign per heartbeat when assignmultiple is turned on. When turned on, the scheduler allocates roughly half of the remaining resources overriding any max.assign settings configured. This is turned ON by default.
+
+
+---
+
+* [YARN-5132](https://issues.apache.org/jira/browse/YARN-5132) | *Critical* | **Exclude generated protobuf sources from YARN Javadoc build**
+
+Exclude javadocs for proto-generated java classes.
+
+
+---
+
+* [HADOOP-13105](https://issues.apache.org/jira/browse/HADOOP-13105) | *Major* | **Support timeouts in LDAP queries in LdapGroupsMapping.**
+
+This patch adds two new config keys for supporting timeouts in LDAP query operations. The property "hadoop.security.group.mapping.ldap.connection.timeout.ms" is the connection timeout (in milliseconds), within which period if the LDAP provider doesn't establish a connection, it will abort the connect attempt. The property "hadoop.security.group.mapping.ldap.read.timeout.ms" is the read timeout (in milliseconds), within which period if the LDAP provider doesn't get a LDAP response, it will abort the read attempt.
+
+
+---
+
+* [HADOOP-13155](https://issues.apache.org/jira/browse/HADOOP-13155) | *Major* | **Implement TokenRenewer to renew and cancel delegation tokens in KMS**
+
+Enables renewal and cancellation of KMS delegation tokens. hadoop.security.key.provider.path needs to be configured to reach the key provider.
+
+
+---
+
+* [HADOOP-12807](https://issues.apache.org/jira/browse/HADOOP-12807) | *Minor* | **S3AFileSystem should read AWS credentials from environment variables**
+
+Adds support to S3AFileSystem for reading AWS credentials from environment variables.
+
+
+---
+
+* [HDFS-10375](https://issues.apache.org/jira/browse/HDFS-10375) | *Trivial* | **Remove redundant TestMiniDFSCluster.testDualClusters**
+
+Remove redundent TestMiniDFSCluster.testDualClusters to save time.
+
+
+---
+
+* [HDFS-10220](https://issues.apache.org/jira/browse/HDFS-10220) | *Major* | **A large number of expired leases can make namenode unresponsive and cause failover**
+
+Two new configuration have been added "dfs.namenode.lease-recheck-interval-ms" and "dfs.namenode.max-lock-hold-to-release-lease-ms" to fine tune the duty cycle with which the Namenode recovers old leases.
+
+
+---
+
+* [HADOOP-13237](https://issues.apache.org/jira/browse/HADOOP-13237) | *Minor* | **s3a initialization against public bucket fails if caller lacks any credentials**
+
+S3A now supports read access to a public S3 bucket even if the client does not configure any AWS credentials. See the documentation of configuration property fs.s3a.aws.credentials.provider for further details.
+
+
+---
+
+* [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | *Minor* | **S3A to support Amazon STS temporary credentials**
+
+S3A now supports use of AWS Security Token Service temporary credentials for authentication to S3. Refer to the documentation of configuration property fs.s3a.session.token for further details.
+
+
+---
+
+* [HADOOP-13175](https://issues.apache.org/jira/browse/HADOOP-13175) | *Major* | **Remove hadoop-ant from hadoop-tools**
+
+The hadoop-ant module in hadoop-tools has been removed.
+
+
+---
+
+* [HADOOP-12666](https://issues.apache.org/jira/browse/HADOOP-12666) | *Major* | **Support Microsoft Azure Data Lake - as a file system in Hadoop**
+
+Hadoop now supports integration with Azure Data Lake as an alternative Hadoop-compatible file system. Please refer to the Hadoop site documentation of Azure Data Lake for details on usage and configuration.
+
+
+---
+
+* [HDFS-9016](https://issues.apache.org/jira/browse/HDFS-9016) | *Major* | **Display upgrade domain information in fsck**
+
+New fsck option "-upgradedomains" has been added to display upgrade domains of any block.
+
+
+---
+
+* [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | *Blocker* | **fix/rewrite create-release**
+
+This rewrites the release process with a new dev-support/bin/create-release script. See http://wiki.apache.org/hadoop/HowToRelease for updated instructions on how to use it.
+
+
+---
+
+* [HADOOP-3733](https://issues.apache.org/jira/browse/HADOOP-3733) | *Minor* | **"s3:" URLs break when Secret Key contains a slash, even if encoded**
+
+Allows userinfo component of URI authority to contain a slash (escaped as %2F). Especially useful for accessing AWS S3 with distcp or hadoop fs.
+
+
+---
+
+* [HADOOP-13242](https://issues.apache.org/jira/browse/HADOOP-13242) | *Major* | **Authenticate to Azure Data Lake using client ID and keys**
+
+Adds support for Azure ActiveDirectory tokens using client ID and keys
+
+
+---
+
+* [HADOOP-9613](https://issues.apache.org/jira/browse/HADOOP-9613) | *Major* | **[JDK8] Update jersey version to latest 1.x release**
+
+Upgrading Jersey and its related libraries:
+
+1. Upgrading jersey from 1.9 to 1.19
+2. Adding jersey-servlet 1.19
+3. Upgrading grizzly-http-servlet from 2.1.2 to 2.2.21
+4. Adding grizzly-http 2.2.21
+5. Adding grizzly-http-server 2.2.21
+
+After upgrading Jersey from 1.12 to 1.13, the root element whose content is empty collection is changed from null to empty object({}).
+
+
+---
+
+* [HDFS-10328](https://issues.apache.org/jira/browse/HDFS-10328) | *Minor* | **Add per-cache-pool default replication num configuration**
+
+Add per-cache-pool default replication num configuration
+
+
+---
+
+* [HADOOP-13203](https://issues.apache.org/jira/browse/HADOOP-13203) | *Major* | **S3A: Support fadvise "random" mode for high performance readPositioned() reads**
+
+S3A has added support for configurable input policies. Similar to fadvise, this configuration provides applications with a way to specify their expected access pattern (sequential or random) while reading a file. S3A then performs optimizations tailored to that access pattern. See site documentation of the fs.s3a.experimental.input.fadvise configuration property for more details. Please be advised that this feature is experimental and subject to backward-incompatible changes in future releases.
+
+
+---
+
+* [HDFS-1312](https://issues.apache.org/jira/browse/HDFS-1312) | *Major* | **Re-balance disks within a Datanode**
+
+The Disk Balancer lets administrators rebalance data across multiple disks of a DataNode. It is useful to correct skewed data distribution often seen after adding or replacing disks. Disk Balancer can be enabled by setting dfs.disk.balancer.enabled to true in hdfs-site.xml. It can be invoked by running "hdfs diskbalancer". See the "HDFS Diskbalancer" section in the HDFS Commands guide for detailed usage.
+
+
+---
+
+* [HADOOP-13263](https://issues.apache.org/jira/browse/HADOOP-13263) | *Major* | **Reload cached groups in background after expiry**
+
+hadoop.security.groups.cache.background.reload can be set to true to enable background reload of expired groups cache entries. This setting can improve the performance of services that use Groups.java (e.g. the NameNode) when group lookups are slow. The setting is disabled by default.
+
+
+---
+
+* [HDFS-10440](https://issues.apache.org/jira/browse/HDFS-10440) | *Major* | **Improve DataNode web UI**
+
+DataNode Web UI has been improved with new HTML5 page, showing useful information.
+
+
+---
+
+* [HADOOP-13209](https://issues.apache.org/jira/browse/HADOOP-13209) | *Major* | **replace slaves with workers**
+
+The 'slaves' file has been deprecated in favor of the 'workers' file and, other than the deprecation warnings, all references to slavery have been removed from the source tree.
+
+
+---
+
+* [HDFS-6434](https://issues.apache.org/jira/browse/HDFS-6434) | *Minor* | **Default permission for creating file should be 644 for WebHdfs/HttpFS**
+
+The default permissions of files and directories created via WebHDFS and HttpFS are now 644 and 755 respectively. See HDFS-10488 for related discussion.
+
+
+---
+
+* [HADOOP-12864](https://issues.apache.org/jira/browse/HADOOP-12864) | *Blocker* | **Remove bin/rcc script**
+
+The rcc command has been removed. See HADOOP-12485 where unused Hadoop Streaming classes were removed.
+
+
+---
+
+* [HADOOP-12709](https://issues.apache.org/jira/browse/HADOOP-12709) | *Major* | **Cut s3:// from trunk**
+
+The s3 file system has been removed. The s3a file system should be used instead.
+
+
+---
+
+* [HADOOP-12064](https://issues.apache.org/jira/browse/HADOOP-12064) | *Blocker* | **[JDK8] Update guice version to 4.0**
+
+Upgrading following dependences:
+\* Guice from 3.0 to 4.0
+\* cglib from 2.2 to 3.2.0
+\* asm from 3.2 to 5.0.4
+
+
+---
+
+* [HDFS-10548](https://issues.apache.org/jira/browse/HDFS-10548) | *Major* | **Remove the long deprecated BlockReaderRemote**
+
+This removes the configuration property {{dfs.client.use.legacy.blockreader}}, since the legacy remote block reader class has been removed from the codebase.
+
+
+---
+
+* [YARN-2928](https://issues.apache.org/jira/browse/YARN-2928) | *Critical* | **YARN Timeline Service v.2: alpha 1**
+
+We are introducing an early preview (alpha 1) of a major revision of YARN Timeline Service: v.2. YARN Timeline Service v.2 addresses two major challenges: improving scalability and reliability of Timeline Service, and enhancing usability by introducing flows and aggregation.
+
+YARN Timeline Service v.2 alpha 1 is provided so that users and developers can test it and provide feedback and suggestions for making it a ready replacement for Timeline Service v.1.x. It should be used only in a test capacity. Most importantly, security is not enabled. Do not set up or use Timeline Service v.2 until security is implemented if security is a critical requirement.
+
+More details are available in the [YARN Timeline Service v.2](./hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html) documentation.
+
+
+---
+
+* [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | *Major* | **remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects**
+
+Dependencies on commons-httpclien
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/43] hadoop git commit: HADOOP-13218. Migrate other Hadoop side
tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and
Kai Zheng
Posted by ae...@apache.org.
HADOOP-13218. Migrate other Hadoop side tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62a96671
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62a96671
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62a96671
Branch: refs/heads/HDFS-7240
Commit: 62a9667136ebd8a048f556b534fcff4fdaf8e2ec
Parents: f955712
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Sep 7 17:05:33 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Sep 7 17:05:33 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 15 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 4 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 +--
.../hadoop/ipc/TestMultipleProtocolServer.java | 236 +--------------
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 -
.../apache/hadoop/ipc/TestRPCCompatibility.java | 242 ++-------------
.../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 ++-
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +++-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 74 +++--
.../hadoop/security/TestDoAsEffectiveUser.java | 291 +++++++------------
.../security/TestUserGroupInformation.java | 28 +-
.../hadoop-common/src/test/proto/test.proto | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 -
.../TestClientProtocolWithDelegationToken.java | 119 --------
.../mapreduce/v2/hs/server/HSAdminServer.java | 3 -
18 files changed, 294 insertions(+), 876 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index 83e4b9e..e68bfd4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -60,7 +60,7 @@ public class ProtobufRpcEngine implements RpcEngine {
private static final ThreadLocal<AsyncGet<Message, Exception>>
ASYNC_RETURN_MESSAGE = new ThreadLocal<>();
- static { // Register the rpcRequest deserializer for WritableRpcEngine
+ static { // Register the rpcRequest deserializer for ProtobufRpcEngine
org.apache.hadoop.ipc.Server.registerProtocolEngine(
RPC.RpcKind.RPC_PROTOCOL_BUFFER, RpcProtobufRequest.class,
new Server.ProtoBufRpcInvoker());
@@ -194,7 +194,8 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (args.length != 2) { // RpcController + Message
- throw new ServiceException("Too many parameters for request. Method: ["
+ throw new ServiceException(
+ "Too many or few parameters for request. Method: ["
+ method.getName() + "]" + ", Expected: 2, Actual: "
+ args.length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 3f68d63..12a07a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ipc;
+import java.io.IOException;
+import java.io.InterruptedIOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
@@ -26,7 +28,6 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.NoRouteToHostException;
import java.net.SocketTimeoutException;
-import java.io.*;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -37,11 +38,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
-import org.apache.commons.logging.*;
-
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolInfoService;
@@ -54,7 +56,6 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
@@ -87,7 +88,7 @@ public class RPC {
RPC_WRITABLE ((short) 2), // Use WritableRpcEngine
RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
- public final short value; //TODO make it private
+ private final short value;
RpcKind(short val) {
this.value = val;
@@ -207,7 +208,7 @@ public class RPC {
RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
if (engine == null) {
Class<?> impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
- WritableRpcEngine.class);
+ ProtobufRpcEngine.class);
engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
PROTOCOL_ENGINES.put(protocol, engine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 80b3303..ea4f533 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -236,14 +236,14 @@ public abstract class Server {
static class RpcKindMapValue {
final Class<? extends Writable> rpcRequestWrapperClass;
final RpcInvoker rpcInvoker;
+
RpcKindMapValue (Class<? extends Writable> rpcRequestWrapperClass,
RpcInvoker rpcInvoker) {
this.rpcInvoker = rpcInvoker;
this.rpcRequestWrapperClass = rpcRequestWrapperClass;
}
}
- static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new
- HashMap<RPC.RpcKind, RpcKindMapValue>(4);
+ static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new HashMap<>(4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 24c75be..fe6fbe4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -715,7 +715,7 @@ public class UserGroupInformation {
*
* @param user The principal name to load from the ticket
* cache
- * @param ticketCachePath the path to the ticket cache file
+ * @param ticketCache the path to the ticket cache file
*
* @throws IOException if the kerberos login fails
*/
@@ -775,7 +775,7 @@ public class UserGroupInformation {
/**
* Create a UserGroupInformation from a Subject with Kerberos principal.
*
- * @param user The KerberosPrincipal to use in UGI
+ * @param subject The KerberosPrincipal to use in UGI
*
* @throws IOException if the kerberos login fails
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index eb7b949..9356dab 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -17,13 +17,8 @@
*/
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.atomic.AtomicLong;
-
+import com.google.common.base.Joiner;
+import com.google.protobuf.BlockingService;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -34,7 +29,6 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoResponseProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
@@ -45,8 +39,12 @@ import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import com.google.common.base.Joiner;
-import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.atomic.AtomicLong;
/**
* Benchmark for protobuf RPC.
@@ -68,7 +66,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
public int secondsToRun = 15;
private int msgSize = 1024;
public Class<? extends RpcEngine> rpcEngine =
- WritableRpcEngine.class;
+ ProtobufRpcEngine.class;
private MyOptions(String args[]) {
try {
@@ -135,7 +133,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
opts.addOption(
OptionBuilder.withLongOpt("engine").hasArg(true)
- .withArgName("writable|protobuf")
+ .withArgName("protobuf")
.withDescription("engine to use")
.create('e'));
@@ -184,8 +182,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
String eng = line.getOptionValue('e');
if ("protobuf".equals(eng)) {
rpcEngine = ProtobufRpcEngine.class;
- } else if ("writable".equals(eng)) {
- rpcEngine = WritableRpcEngine.class;
} else {
throw new ParseException("invalid engine: " + eng);
}
@@ -237,11 +233,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
.setInstance(service).setBindAddress(opts.host).setPort(opts.getPort())
.setNumHandlers(opts.serverThreads).setVerbose(false).build();
- } else if (opts.rpcEngine == WritableRpcEngine.class) {
- server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestRPC.TestImpl()).setBindAddress(opts.host)
- .setPort(opts.getPort()).setNumHandlers(opts.serverThreads)
- .setVerbose(false).build();
} else {
throw new RuntimeException("Bad engine: " + opts.rpcEngine);
}
@@ -399,15 +390,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
return responseProto.getMessage();
}
};
- } else if (opts.rpcEngine == WritableRpcEngine.class) {
- final TestProtocol proxy = RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID, addr, conf);
- return new RpcServiceWrapper() {
- @Override
- public String doEcho(String msg) throws Exception {
- return proxy.echo(msg);
- }
- };
} else {
throw new RuntimeException("unsupported engine: " + opts.rpcEngine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
index 8b419e3..10e23ba 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
@@ -17,252 +17,28 @@
*/
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
-import org.apache.hadoop.net.NetUtils;
-import org.junit.Before;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
-import com.google.protobuf.BlockingService;
public class TestMultipleProtocolServer extends TestRpcBase {
- private static InetSocketAddress addr;
- private static RPC.Server server;
-
- private static Configuration conf = new Configuration();
-
-
- @ProtocolInfo(protocolName="Foo")
- interface Foo0 extends VersionedProtocol {
- public static final long versionID = 0L;
- String ping() throws IOException;
-
- }
-
- @ProtocolInfo(protocolName="Foo")
- interface Foo1 extends VersionedProtocol {
- public static final long versionID = 1L;
- String ping() throws IOException;
- String ping2() throws IOException;
- }
-
- @ProtocolInfo(protocolName="Foo")
- interface FooUnimplemented extends VersionedProtocol {
- public static final long versionID = 2L;
- String ping() throws IOException;
- }
-
- interface Mixin extends VersionedProtocol{
- public static final long versionID = 0L;
- void hello() throws IOException;
- }
-
- interface Bar extends Mixin {
- public static final long versionID = 0L;
- int echo(int i) throws IOException;
- }
-
- class Foo0Impl implements Foo0 {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Foo0.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public String ping() {
- return "Foo0";
- }
-
- }
-
- class Foo1Impl implements Foo1 {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Foo1.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public String ping() {
- return "Foo1";
- }
- @Override
- public String ping2() {
- return "Foo1";
-
- }
-
- }
-
-
- class BarImpl implements Bar {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Bar.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public int echo(int i) {
- return i;
- }
-
- @Override
- public void hello() {
+ private static RPC.Server server;
-
- }
- }
@Before
public void setUp() throws Exception {
- // create a server with two handlers
- server = new RPC.Builder(conf).setProtocol(Foo0.class)
- .setInstance(new Foo0Impl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Foo1.class, new Foo1Impl());
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Bar.class, new BarImpl());
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Mixin.class, new BarImpl());
-
-
- // Add Protobuf server
- // Create server side implementation
- PBServerImpl pbServerImpl = new PBServerImpl();
- BlockingService service = TestProtobufRpcProto
- .newReflectiveBlockingService(pbServerImpl);
- server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService.class,
- service);
- server.start();
- addr = NetUtils.getConnectAddress(server);
+ super.setupConf();
+
+ server = setupTestServer(conf, 2);
}
-
+
@After
public void tearDown() throws Exception {
server.stop();
}
- @Test
- public void test1() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(Foo0.class, Foo0.versionID, addr, conf);
- Foo0 foo0 = (Foo0)proxy.getProxy();
- Assert.assertEquals("Foo0", foo0.ping());
-
-
- proxy = RPC.getProtocolProxy(Foo1.class, Foo1.versionID, addr, conf);
-
-
- Foo1 foo1 = (Foo1)proxy.getProxy();
- Assert.assertEquals("Foo1", foo1.ping());
- Assert.assertEquals("Foo1", foo1.ping());
-
-
- proxy = RPC.getProtocolProxy(Bar.class, Foo1.versionID, addr, conf);
-
-
- Bar bar = (Bar)proxy.getProxy();
- Assert.assertEquals(99, bar.echo(99));
-
- // Now test Mixin class method
-
- Mixin mixin = bar;
- mixin.hello();
- }
-
-
- // Server does not implement the FooUnimplemented version of protocol Foo.
- // See that calls to it fail.
- @Test(expected=IOException.class)
- public void testNonExistingProtocol() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(FooUnimplemented.class,
- FooUnimplemented.versionID, addr, conf);
-
- FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
- foo.ping();
- }
-
- /**
- * getProtocolVersion of an unimplemented version should return highest version
- * Similarly getProtocolSignature should work.
- * @throws IOException
- */
- @Test
- public void testNonExistingProtocol2() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(FooUnimplemented.class,
- FooUnimplemented.versionID, addr, conf);
-
- FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
- Assert.assertEquals(Foo1.versionID,
- foo.getProtocolVersion(RPC.getProtocolName(FooUnimplemented.class),
- FooUnimplemented.versionID));
- foo.getProtocolSignature(RPC.getProtocolName(FooUnimplemented.class),
- FooUnimplemented.versionID, 0);
- }
-
- @Test(expected=IOException.class)
- public void testIncorrectServerCreation() throws IOException {
- new RPC.Builder(conf).setProtocol(Foo1.class).setInstance(new Foo0Impl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(2).setVerbose(false)
- .build();
- }
-
// Now test a PB service - a server hosts both PB and Writable Rpcs.
@Test
public void testPBService() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
index 969f728..6d83d7d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
@@ -26,19 +26,6 @@ import org.junit.Test;
public class TestRPCCallBenchmark {
@Test(timeout=20000)
- public void testBenchmarkWithWritable() throws Exception {
- int rc = ToolRunner.run(new RPCCallBenchmark(),
- new String[] {
- "--clientThreads", "30",
- "--serverThreads", "30",
- "--time", "5",
- "--serverReaderThreads", "4",
- "--messageSize", "1024",
- "--engine", "writable"});
- assertEquals(0, rc);
- }
-
- @Test(timeout=20000)
public void testBenchmarkWithProto() throws Exception {
int rc = ToolRunner.run(new RPCCallBenchmark(),
new String[] {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
index 2ac2be9..a06d9fd 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
@@ -18,28 +18,20 @@
package org.apache.hadoop.ipc;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
-import org.apache.hadoop.net.NetUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
/** Unit test for supporting method-name based compatible RPCs. */
public class TestRPCCompatibility {
private static final String ADDRESS = "0.0.0.0";
@@ -49,7 +41,7 @@ public class TestRPCCompatibility {
public static final Log LOG =
LogFactory.getLog(TestRPCCompatibility.class);
-
+
private static Configuration conf = new Configuration();
public interface TestProtocol0 extends VersionedProtocol {
@@ -120,6 +112,21 @@ public class TestRPCCompatibility {
@Before
public void setUp() {
ProtocolSignature.resetCache();
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol0.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol1.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol2.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol3.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol4.class, ProtobufRpcEngine.class);
}
@After
@@ -133,117 +140,7 @@ public class TestRPCCompatibility {
server = null;
}
}
-
- @Test // old client vs new server
- public void testVersion0ClientVersion1Server() throws Exception {
- // create a server with two handlers
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- proxy = RPC.getProtocolProxy(
- TestProtocol0.class, TestProtocol0.versionID, addr, conf);
-
- TestProtocol0 proxy0 = (TestProtocol0)proxy.getProxy();
- proxy0.ping();
- }
-
- @Test // old client vs new server
- public void testVersion1ClientVersion0Server() throws Exception {
- // create a server with two handlers
- server = new RPC.Builder(conf).setProtocol(TestProtocol0.class)
- .setInstance(new TestImpl0()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- proxy = RPC.getProtocolProxy(
- TestProtocol1.class, TestProtocol1.versionID, addr, conf);
-
- TestProtocol1 proxy1 = (TestProtocol1)proxy.getProxy();
- proxy1.ping();
- try {
- proxy1.echo("hello");
- fail("Echo should fail");
- } catch(IOException e) {
- }
- }
-
- private class Version2Client {
- private TestProtocol2 proxy2;
- private ProtocolProxy<TestProtocol2> serverInfo;
-
- private Version2Client() throws IOException {
- serverInfo = RPC.getProtocolProxy(
- TestProtocol2.class, TestProtocol2.versionID, addr, conf);
- proxy2 = serverInfo.getProxy();
- }
-
- public int echo(int value) throws IOException, NumberFormatException {
- if (serverInfo.isMethodSupported("echo", int.class)) {
-System.out.println("echo int is supported");
- return -value; // use version 3 echo long
- } else { // server is version 2
-System.out.println("echo int is NOT supported");
- return Integer.parseInt(proxy2.echo(String.valueOf(value)));
- }
- }
-
- public String echo(String value) throws IOException {
- return proxy2.echo(value);
- }
-
- public void ping() throws IOException {
- proxy2.ping();
- }
- }
-
- @Test // Compatible new client & old server
- public void testVersion2ClientVersion1Server() throws Exception {
- // create a server with two handlers
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
-
- Version2Client client = new Version2Client();
- client.ping();
- assertEquals("hello", client.echo("hello"));
-
- // echo(int) is not supported by server, so returning 3
- // This verifies that echo(int) and echo(String)'s hash codes are different
- assertEquals(3, client.echo(3));
- }
-
- @Test // equal version client and server
- public void testVersion2ClientVersion2Server() throws Exception {
- // create a server with two handlers
- TestImpl2 impl = new TestImpl2();
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- Version2Client client = new Version2Client();
-
- client.ping();
- assertEquals("hello", client.echo("hello"));
-
- // now that echo(int) is supported by the server, echo(int) should return -3
- assertEquals(-3, client.echo(3));
- }
-
public interface TestProtocol3 {
int echo(String value);
int echo(int value);
@@ -297,97 +194,4 @@ System.out.println("echo int is NOT supported");
@Override
int echo(int value) throws IOException;
}
-
- @Test
- public void testVersionMismatch() throws IOException {
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- TestProtocol4 proxy = RPC.getProxy(TestProtocol4.class,
- TestProtocol4.versionID, addr, conf);
- try {
- proxy.echo(21);
- fail("The call must throw VersionMismatch exception");
- } catch (RemoteException ex) {
- Assert.assertEquals(RPC.VersionMismatch.class.getName(),
- ex.getClassName());
- Assert.assertTrue(ex.getErrorCode().equals(
- RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH));
- } catch (IOException ex) {
- fail("Expected version mismatch but got " + ex);
- }
- }
-
- @Test
- public void testIsMethodSupported() throws IOException {
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- TestProtocol2 proxy = RPC.getProxy(TestProtocol2.class,
- TestProtocol2.versionID, addr, conf);
- boolean supported = RpcClientUtil.isMethodSupported(proxy,
- TestProtocol2.class, RPC.RpcKind.RPC_WRITABLE,
- RPC.getProtocolVersion(TestProtocol2.class), "echo");
- Assert.assertTrue(supported);
- supported = RpcClientUtil.isMethodSupported(proxy,
- TestProtocol2.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
- RPC.getProtocolVersion(TestProtocol2.class), "echo");
- Assert.assertFalse(supported);
- }
-
- /**
- * Verify that ProtocolMetaInfoServerSideTranslatorPB correctly looks up
- * the server registry to extract protocol signatures and versions.
- */
- @Test
- public void testProtocolMetaInfoSSTranslatorPB() throws Exception {
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
-
- ProtocolMetaInfoServerSideTranslatorPB xlator =
- new ProtocolMetaInfoServerSideTranslatorPB(server);
-
- GetProtocolSignatureResponseProto resp = xlator.getProtocolSignature(
- null,
- createGetProtocolSigRequestProto(TestProtocol1.class,
- RPC.RpcKind.RPC_PROTOCOL_BUFFER));
- //No signatures should be found
- Assert.assertEquals(0, resp.getProtocolSignatureCount());
- resp = xlator.getProtocolSignature(
- null,
- createGetProtocolSigRequestProto(TestProtocol1.class,
- RPC.RpcKind.RPC_WRITABLE));
- Assert.assertEquals(1, resp.getProtocolSignatureCount());
- ProtocolSignatureProto sig = resp.getProtocolSignatureList().get(0);
- Assert.assertEquals(TestProtocol1.versionID, sig.getVersion());
- boolean found = false;
- int expected = ProtocolSignature.getFingerprint(TestProtocol1.class
- .getMethod("echo", String.class));
- for (int m : sig.getMethodsList()) {
- if (expected == m) {
- found = true;
- break;
- }
- }
- Assert.assertTrue(found);
- }
-
- private GetProtocolSignatureRequestProto createGetProtocolSigRequestProto(
- Class<?> protocol, RPC.RpcKind rpcKind) {
- GetProtocolSignatureRequestProto.Builder builder =
- GetProtocolSignatureRequestProto.newBuilder();
- builder.setProtocol(protocol.getName());
- builder.setRpcKind(rpcKind.toString());
- return builder.build();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
index 5807998..b22f91b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ipc;
import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
-import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -30,11 +28,13 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.nio.channels.ClosedByInterruptException;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
+
/**
* tests that the proxy can be interrupted
*/
-public class TestRPCWaitForProxy extends Assert {
- private static final String ADDRESS = "0.0.0.0";
+public class TestRPCWaitForProxy extends TestRpcBase {
private static final Logger
LOG = LoggerFactory.getLogger(TestRPCWaitForProxy.class);
@@ -46,14 +46,15 @@ public class TestRPCWaitForProxy extends Assert {
*
* @throws Throwable any exception other than that which was expected
*/
- @Test(timeout = 10000)
+ @Test(timeout = 50000)
public void testWaitForProxy() throws Throwable {
RpcThread worker = new RpcThread(0);
worker.start();
worker.join();
Throwable caught = worker.getCaught();
- assertNotNull("No exception was raised", caught);
- if (!(caught instanceof ConnectException)) {
+ Throwable cause = caught.getCause();
+ Assert.assertNotNull("No exception was raised", cause);
+ if (!(cause instanceof ConnectException)) {
throw caught;
}
}
@@ -69,11 +70,11 @@ public class TestRPCWaitForProxy extends Assert {
RpcThread worker = new RpcThread(100);
worker.start();
Thread.sleep(1000);
- assertTrue("worker hasn't started", worker.waitStarted);
+ Assert.assertTrue("worker hasn't started", worker.waitStarted);
worker.interrupt();
worker.join();
Throwable caught = worker.getCaught();
- assertNotNull("No exception was raised", caught);
+ Assert.assertNotNull("No exception was raised", caught);
// looking for the root cause here, which can be wrapped
// as part of the NetUtils work. Having this test look
// a the type of exception there would be brittle to improvements
@@ -82,6 +83,8 @@ public class TestRPCWaitForProxy extends Assert {
if (cause == null) {
// no inner cause, use outer exception as root cause.
cause = caught;
+ } else if (cause.getCause() != null) {
+ cause = cause.getCause();
}
if (!(cause instanceof InterruptedIOException)
&& !(cause instanceof ClosedByInterruptException)) {
@@ -112,12 +115,16 @@ public class TestRPCWaitForProxy extends Assert {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
connectRetries);
waitStarted = true;
- TestProtocol proxy = RPC.waitForProxy(TestProtocol.class,
- TestProtocol.versionID,
- new InetSocketAddress(ADDRESS, 20),
- config,
- 15000L);
- proxy.echo("");
+
+ short invalidPort = 20;
+ InetSocketAddress invalidAddress = new InetSocketAddress(ADDRESS,
+ invalidPort);
+ TestRpcBase.TestRpcService proxy = RPC.getProxy(
+ TestRpcBase.TestRpcService.class,
+ 1L, invalidAddress, conf);
+ // Test echo method
+ proxy.echo(null, newEchoRequest("hello"));
+
} catch (Throwable throwable) {
caught = throwable;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
index bc604a4..5a8f8d0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -112,7 +112,8 @@ public class TestRpcBase {
return setupTestServer(builder);
}
- protected static RPC.Server setupTestServer(RPC.Builder builder) throws IOException {
+ protected static RPC.Server setupTestServer(
+ RPC.Builder builder) throws IOException {
RPC.Server server = builder.build();
server.start();
@@ -175,17 +176,21 @@ public class TestRpcBase {
public TestTokenIdentifier() {
this(new Text(), new Text());
}
+
public TestTokenIdentifier(Text tokenid) {
this(tokenid, new Text());
}
+
public TestTokenIdentifier(Text tokenid, Text realUser) {
this.tokenid = tokenid == null ? new Text() : tokenid;
this.realUser = realUser == null ? new Text() : realUser;
}
+
@Override
public Text getKind() {
return KIND_NAME;
}
+
@Override
public UserGroupInformation getUser() {
if (realUser.toString().isEmpty()) {
@@ -203,6 +208,7 @@ public class TestRpcBase {
tokenid.readFields(in);
realUser.readFields(in);
}
+
@Override
public void write(DataOutput out) throws IOException {
tokenid.write(out);
@@ -234,7 +240,7 @@ public class TestRpcBase {
@SuppressWarnings("unchecked")
@Override
public Token<TestTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
+ Collection<Token<? extends TokenIdentifier>> tokens) {
if (service == null) {
return null;
}
@@ -388,19 +394,17 @@ public class TestRpcBase {
}
@Override
- public TestProtos.AuthUserResponseProto getAuthUser(
+ public TestProtos.UserResponseProto getAuthUser(
RpcController controller, TestProtos.EmptyRequestProto request)
throws ServiceException {
- UserGroupInformation authUser = null;
+ UserGroupInformation authUser;
try {
authUser = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new ServiceException(e);
}
- return TestProtos.AuthUserResponseProto.newBuilder()
- .setAuthUser(authUser.getUserName())
- .build();
+ return newUserResponse(authUser.getUserName());
}
@Override
@@ -432,6 +436,34 @@ public class TestRpcBase {
return TestProtos.EmptyResponseProto.newBuilder().build();
}
+
+ @Override
+ public TestProtos.UserResponseProto getCurrentUser(
+ RpcController controller,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
+ String user;
+ try {
+ user = UserGroupInformation.getCurrentUser().toString();
+ } catch (IOException e) {
+ throw new ServiceException("Failed to get current user", e);
+ }
+
+ return newUserResponse(user);
+ }
+
+ @Override
+ public TestProtos.UserResponseProto getServerRemoteUser(
+ RpcController controller,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
+ String serverRemoteUser = Server.getRemoteUser().toString();
+ return newUserResponse(serverRemoteUser);
+ }
+
+ private TestProtos.UserResponseProto newUserResponse(String user) {
+ return TestProtos.UserResponseProto.newBuilder()
+ .setUser(user)
+ .build();
+ }
}
protected static TestProtos.EmptyRequestProto newEmptyRequest() {
@@ -478,8 +510,4 @@ public class TestRpcBase {
}
return null;
}
-
- protected static String convert(TestProtos.AuthUserResponseProto response) {
- return response.getAuthUser();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index 72371a7..c48ff2e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -45,30 +45,55 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import javax.security.auth.callback.*;
-import javax.security.sasl.*;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.sasl.AuthorizeCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslClient;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.security.Security;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.*;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.KERBEROS;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.SIMPLE;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.TOKEN;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/** Unit tests for using Sasl over RPC. */
@RunWith(Parameterized.class)
public class TestSaslRPC extends TestRpcBase {
@Parameters
public static Collection<Object[]> data() {
- Collection<Object[]> params = new ArrayList<Object[]>();
+ Collection<Object[]> params = new ArrayList<>();
for (QualityOfProtection qop : QualityOfProtection.values()) {
params.add(new Object[]{ new QualityOfProtection[]{qop},qop, null });
}
@@ -114,7 +139,7 @@ public class TestSaslRPC extends TestRpcBase {
NONE(),
VALID(),
INVALID(),
- OTHER();
+ OTHER()
}
@BeforeClass
@@ -230,7 +255,7 @@ public class TestSaslRPC extends TestRpcBase {
final Server server = setupTestServer(conf, 5, sm);
doDigestRpc(server, sm);
} finally {
- SecurityUtil.setSecurityInfoProviders(new SecurityInfo[0]);
+ SecurityUtil.setSecurityInfoProviders();
}
}
@@ -259,7 +284,7 @@ public class TestSaslRPC extends TestRpcBase {
addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId, sm);
+ Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
@@ -296,8 +321,8 @@ public class TestSaslRPC extends TestRpcBase {
// set doPing to true
newConf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
- ConnectionId remoteId = ConnectionId.getConnectionId(new InetSocketAddress(0),
- TestRpcService.class, null, 0, null, newConf);
+ ConnectionId remoteId = ConnectionId.getConnectionId(
+ new InetSocketAddress(0), TestRpcService.class, null, 0, null, newConf);
assertEquals(CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT,
remoteId.getPingInterval());
// set doPing to false
@@ -806,13 +831,13 @@ public class TestSaslRPC extends TestRpcBase {
final TestTokenSecretManager sm = new TestTokenSecretManager();
boolean useSecretManager = (serverAuth != SIMPLE);
if (enableSecretManager != null) {
- useSecretManager &= enableSecretManager.booleanValue();
+ useSecretManager &= enableSecretManager;
}
if (forceSecretManager != null) {
- useSecretManager |= forceSecretManager.booleanValue();
+ useSecretManager |= forceSecretManager;
}
final SecretManager<?> serverSm = useSecretManager ? sm : null;
-
+
Server server = serverUgi.doAs(new PrivilegedExceptionAction<Server>() {
@Override
public Server run() throws IOException {
@@ -867,13 +892,13 @@ public class TestSaslRPC extends TestRpcBase {
proxy.ping(null, newEmptyRequest());
// make sure the other side thinks we are who we said we are!!!
assertEquals(clientUgi.getUserName(),
- convert(proxy.getAuthUser(null, newEmptyRequest())));
+ proxy.getAuthUser(null, newEmptyRequest()).getUser());
AuthMethod authMethod =
convert(proxy.getAuthMethod(null, newEmptyRequest()));
// verify sasl completed with correct QOP
assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null,
- RPC.getConnectionIdForProxy(proxy).getSaslQop());
- return authMethod.toString();
+ RPC.getConnectionIdForProxy(proxy).getSaslQop());
+ return authMethod != null ? authMethod.toString() : null;
} catch (ServiceException se) {
if (se.getCause() instanceof RemoteException) {
throw (RemoteException) se.getCause();
@@ -898,21 +923,18 @@ public class TestSaslRPC extends TestRpcBase {
String actual) {
assertEquals(expect.toString(), actual);
}
-
- private static void assertAuthEquals(Pattern expect,
- String actual) {
+
+ private static void assertAuthEquals(Pattern expect, String actual) {
// this allows us to see the regexp and the value it didn't match
if (!expect.matcher(actual).matches()) {
- assertEquals(expect, actual); // it failed
- } else {
- assertTrue(true); // it matched
+ fail(); // it failed
}
}
/*
* Class used to test overriding QOP values using SaslPropertiesResolver
*/
- static class AuthSaslPropertiesResolver extends SaslPropertiesResolver{
+ static class AuthSaslPropertiesResolver extends SaslPropertiesResolver {
@Override
public Map<String, String> getServerProperties(InetAddress address) {
@@ -921,7 +943,7 @@ public class TestSaslRPC extends TestRpcBase {
return newPropertes;
}
}
-
+
public static void main(String[] args) throws Exception {
System.out.println("Testing Kerberos authentication over RPC");
if (args.length != 2) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
index 50d389c..c4dbcac 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
@@ -17,40 +17,35 @@
*/
package org.apache.hadoop.security;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.NetworkInterface;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Enumeration;
-
-import org.junit.Assert;
-
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ipc.TestRpcBase;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenInfo;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenSecretManager;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenSelector;
-import org.apache.commons.logging.*;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Enumeration;
/**
- *
+ * Test do as effective user.
*/
-public class TestDoAsEffectiveUser {
+public class TestDoAsEffectiveUser extends TestRpcBase {
final private static String REAL_USER_NAME = "realUser1@HADOOP.APACHE.ORG";
final private static String REAL_USER_SHORT_NAME = "realUser1";
final private static String PROXY_USER_NAME = "proxyUser";
@@ -58,8 +53,8 @@ public class TestDoAsEffectiveUser {
final private static String GROUP2_NAME = "group2";
final private static String[] GROUP_NAMES = new String[] { GROUP1_NAME,
GROUP2_NAME };
- private static final String ADDRESS = "0.0.0.0";
- private TestProtocol proxy;
+
+ private TestRpcService client;
private static final Configuration masterConf = new Configuration();
@@ -82,7 +77,7 @@ public class TestDoAsEffectiveUser {
private void configureSuperUserIPAddresses(Configuration conf,
String superUserShortName) throws IOException {
- ArrayList<String> ipList = new ArrayList<String>();
+ ArrayList<String> ipList = new ArrayList<>();
Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
.getNetworkInterfaces();
while (netInterfaceList.hasMoreElements()) {
@@ -130,50 +125,19 @@ public class TestDoAsEffectiveUser {
curUGI.toString());
}
- @TokenInfo(TestTokenSelector.class)
- public interface TestProtocol extends VersionedProtocol {
- public static final long versionID = 1L;
-
- String aMethod() throws IOException;
- String getServerRemoteUser() throws IOException;
- }
-
- public class TestImpl implements TestProtocol {
-
- @Override
- public String aMethod() throws IOException {
- return UserGroupInformation.getCurrentUser().toString();
- }
-
- @Override
- public String getServerRemoteUser() throws IOException {
- return Server.getRemoteUser().toString();
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return TestProtocol.versionID;
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- return new ProtocolSignature(TestProtocol.versionID, null);
- }
- }
-
- private void checkRemoteUgi(final Server server,
- final UserGroupInformation ugi, final Configuration conf)
- throws Exception {
+ private void checkRemoteUgi(final UserGroupInformation ugi,
+ final Configuration conf) throws Exception {
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws IOException {
- proxy = RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID,
- NetUtils.getConnectAddress(server), conf);
- Assert.assertEquals(ugi.toString(), proxy.aMethod());
- Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser());
+ public Void run() throws ServiceException {
+ client = getClient(addr, conf);
+ String currentUser = client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
+ String serverRemoteUser = client.getServerRemoteUser(null,
+ newEmptyRequest()).getUser();
+
+ Assert.assertEquals(ugi.toString(), currentUser);
+ Assert.assertEquals(ugi.toString(), serverRemoteUser);
return null;
}
});
@@ -185,29 +149,27 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).build();
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(server, realUserUgi, conf);
+ checkRemoteUgi(realUserUgi, conf);
- UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
+ UserGroupInformation proxyUserUgi =
+ UserGroupInformation.createProxyUserForTesting(
PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(server, proxyUserUgi, conf);
+ checkRemoteUgi(proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -218,29 +180,25 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(server, realUserUgi, conf);
+ checkRemoteUgi(realUserUgi, conf);
UserGroupInformation proxyUserUgi = UserGroupInformation
.createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(server, proxyUserUgi, conf);
+ checkRemoteUgi(proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -256,17 +214,14 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -275,11 +230,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -287,10 +241,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -299,17 +250,14 @@ public class TestDoAsEffectiveUser {
final Configuration conf = new Configuration();
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -318,11 +266,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -330,10 +277,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -341,15 +285,12 @@ public class TestDoAsEffectiveUser {
public void testRealUserGroupNotSpecified() throws IOException {
final Configuration conf = new Configuration();
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -358,11 +299,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = (TestProtocol) RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -370,10 +310,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -384,17 +321,14 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group3");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -403,11 +337,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -415,10 +348,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -432,20 +362,17 @@ public class TestDoAsEffectiveUser {
final Configuration conf = new Configuration(masterConf);
TestTokenSecretManager sm = new TestTokenSecretManager();
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(conf);
- final Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
-
- server.start();
+ final Server server = setupTestServer(conf, 5, sm);
final UserGroupInformation current = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
+ Token<TestTokenIdentifier> token = new Token<>(tokenId,
sm);
SecurityUtil.setTokenService(token, addr);
UserGroupInformation proxyUserUgi = UserGroupInformation
@@ -453,23 +380,19 @@ public class TestDoAsEffectiveUser {
proxyUserUgi.addToken(token);
refreshConf(conf);
-
+
String retVal = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
});
@@ -486,42 +409,34 @@ public class TestDoAsEffectiveUser {
TestTokenSecretManager sm = new TestTokenSecretManager();
final Configuration newConf = new Configuration(masterConf);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, newConf);
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(newConf, TestRpcService.class,
+ ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(newConf);
- final Server server = new RPC.Builder(newConf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(sm).build();
-
- server.start();
+ final Server server = setupTestServer(newConf, 5, sm);
final UserGroupInformation current = UserGroupInformation
.createUserForTesting(REAL_USER_NAME, GROUP_NAMES);
refreshConf(newConf);
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
- sm);
+ Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
String retVal = current.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, newConf);
- String ret = proxy.aMethod();
- return ret;
+ client = getClient(addr, newConf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index ef5c6a2..a306d35 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -28,7 +29,11 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -49,9 +54,22 @@ import java.util.Set;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import static org.apache.hadoop.ipc.TestSaslRPC.*;
-import static org.apache.hadoop.test.MetricsAsserts.*;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
+import static org.apache.hadoop.test.MetricsAsserts.assertGaugeGt;
+import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
+import static org.apache.hadoop.test.MetricsAsserts.getDoubleGauge;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -108,7 +126,7 @@ public class TestUserGroupInformation {
UserGroupInformation.setLoginUser(null);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testSimpleLogin() throws IOException {
tryLoginAuthenticationMethod(AuthenticationMethod.SIMPLE, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/proto/test.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test.proto b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
index 99cd93d..6411f97 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
@@ -88,6 +88,6 @@ message AuthMethodResponseProto {
required string mechanismName = 2;
}
-message AuthUserResponseProto {
- required string authUser = 1;
+message UserResponseProto {
+ required string user = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 3292115..06f6c4f 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -40,9 +40,11 @@ service TestProtobufRpcProto {
rpc exchange(ExchangeRequestProto) returns (ExchangeResponseProto);
rpc sleep(SleepRequestProto) returns (EmptyResponseProto);
rpc getAuthMethod(EmptyRequestProto) returns (AuthMethodResponseProto);
- rpc getAuthUser(EmptyRequestProto) returns (AuthUserResponseProto);
+ rpc getAuthUser(EmptyRequestProto) returns (UserResponseProto);
rpc echoPostponed(EchoRequestProto) returns (EchoResponseProto);
rpc sendPostponed(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc getCurrentUser(EmptyRequestProto) returns (UserResponseProto);
+ rpc getServerRemoteUser(EmptyRequestProto) returns (UserResponseProto);
}
service TestProtobufRpc2Proto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 6b52949..57f7cb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -168,7 +168,6 @@ import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
import org.apache.hadoop.net.Node;
@@ -317,8 +316,6 @@ public class NameNodeRpcServer implements NamenodeProtocols {
new TraceAdminProtocolServerSideTranslatorPB(this);
BlockingService traceAdminService = TraceAdminService
.newReflectiveBlockingService(traceAdminXlator);
-
- WritableRpcEngine.ensureInitialized();
InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
if (serviceRpcAddr != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
deleted file mode 100644
index 0b7ee33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.security;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.mockito.Mockito.mock;
-
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SaslInputStream;
-import org.apache.hadoop.security.SaslRpcClient;
-import org.apache.hadoop.security.SaslRpcServer;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
-import org.junit.Test;
-
-/** Unit tests for using Delegation Token over RPC. */
-public class TestClientProtocolWithDelegationToken {
- private static final String ADDRESS = "0.0.0.0";
-
- public static final Log LOG = LogFactory
- .getLog(TestClientProtocolWithDelegationToken.class);
-
- private static final Configuration conf;
- static {
- conf = new Configuration();
- conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- UserGroupInformation.setConfiguration(conf);
- }
-
- static {
- GenericTestUtils.setLogLevel(Client.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(Server.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslRpcClient.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslRpcServer.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslInputStream.LOG, Level.ALL);
- }
-
- @Test
- public void testDelegationTokenRpc() throws Exception {
- ClientProtocol mockNN = mock(ClientProtocol.class);
- FSNamesystem mockNameSys = mock(FSNamesystem.class);
-
- DelegationTokenSecretManager sm = new DelegationTokenSecretManager(
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
- 3600000, mockNameSys);
- sm.startThreads();
- final Server server = new RPC.Builder(conf)
- .setProtocol(ClientProtocol.class).setInstance(mockNN)
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(sm).build();
-
- server.start();
-
- final UserGroupInformation current = UserGroupInformation.getCurrentUser();
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
- String user = current.getUserName();
- Text owner = new Text(user);
- DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null);
- Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
- dtId, sm);
- SecurityUtil.setTokenService(token, addr);
- LOG.info("Service for token is " + token.getService());
- current.addToken(token);
- current.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- ClientProtocol proxy = null;
- try {
- proxy = RPC.getProxy(ClientProtocol.class,
- ClientProtocol.versionID, addr, conf);
- proxy.getServerDefaults();
- } finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
- }
- return null;
- }
- });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
index 3fef5e2..729af0a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
@@ -98,8 +97,6 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
BlockingService refreshHSAdminProtocolService = HSAdminRefreshProtocolService
.newReflectiveBlockingService(refreshHSAdminProtocolXlator);
- WritableRpcEngine.ensureInitialized();
-
clientRpcAddress = conf.getSocketAddr(
JHAdminConfig.MR_HISTORY_BIND_HOST,
JHAdminConfig.JHS_ADMIN_ADDRESS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/43] hadoop git commit: HDFS-10844.
test_libhdfs_threaded_hdfs_static and test_libhdfs_zerocopy_hdfs_static are
failing.
Posted by ae...@apache.org.
HDFS-10844. test_libhdfs_threaded_hdfs_static and test_libhdfs_zerocopy_hdfs_static are failing.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d1bf53c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d1bf53c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d1bf53c
Branch: refs/heads/HDFS-7240
Commit: 2d1bf53c7e208ad951ebed7ee3f2e44582dfd151
Parents: 63f5948
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Sep 9 00:49:22 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Sep 9 00:49:22 2016 +0900
----------------------------------------------------------------------
.../src/main/native/libhdfs/hdfs.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1bf53c/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index 4618dbb..1dcc768 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -114,7 +114,7 @@ int hdfsFileGetReadStatistics(hdfsFile file,
jthr = invokeMethod(env, &jVal, INSTANCE, file->file,
"org/apache/hadoop/hdfs/client/HdfsDataInputStream",
"getReadStatistics",
- "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;");
+ "()Lorg/apache/hadoop/hdfs/ReadStatistics;");
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"hdfsFileGetReadStatistics: getReadStatistics failed");
@@ -127,7 +127,7 @@ int hdfsFileGetReadStatistics(hdfsFile file,
goto done;
}
jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
- "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+ "org/apache/hadoop/hdfs/ReadStatistics",
"getTotalBytesRead", "()J");
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -137,7 +137,7 @@ int hdfsFileGetReadStatistics(hdfsFile file,
s->totalBytesRead = jVal.j;
jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
- "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+ "org/apache/hadoop/hdfs/ReadStatistics",
"getTotalLocalBytesRead", "()J");
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -147,7 +147,7 @@ int hdfsFileGetReadStatistics(hdfsFile file,
s->totalLocalBytesRead = jVal.j;
jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
- "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+ "org/apache/hadoop/hdfs/ReadStatistics",
"getTotalShortCircuitBytesRead", "()J");
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -156,7 +156,7 @@ int hdfsFileGetReadStatistics(hdfsFile file,
}
s->totalShortCircuitBytesRead = jVal.j;
jthr = invokeMethod(env, &jVal, INSTANCE, readStats,
- "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics",
+ "org/apache/hadoop/hdfs/ReadStatistics",
"getTotalZeroCopyBytesRead", "()J");
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/43] hadoop git commit: HDFS-10822. Log DataNodes in the write
pipeline. John Zhuge via Lei Xu
Posted by ae...@apache.org.
HDFS-10822. Log DataNodes in the write pipeline. John Zhuge via Lei Xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a8c5064
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a8c5064
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a8c5064
Branch: refs/heads/HDFS-7240
Commit: 5a8c5064d1a1d596b1f5c385299a86ec6ab9ad1e
Parents: 23abb09
Author: Lei Xu <le...@apache.org>
Authored: Fri Sep 2 11:01:13 2016 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Fri Sep 2 11:01:13 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a8c5064/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 062a416..5166c8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -508,6 +508,12 @@ class DataStreamer extends Daemon {
private void initDataStreaming() {
this.setName("DataStreamer for file " + src +
" block " + block);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("nodes {} storageTypes {} storageIDs {}",
+ Arrays.toString(nodes),
+ Arrays.toString(storageTypes),
+ Arrays.toString(storageIDs));
+ }
response = new ResponseProcessor(nodes);
response.start();
stage = BlockConstructionStage.DATA_STREAMING;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org