You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/07/16 19:36:46 UTC
[01/21] hadoop git commit: Revert "YARN-3878. AsyncDispatcher can
hang while stopping if it is configured for draining events on stop. (Varun
Saxena via kasha)"
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-12111 33f2feb1a -> 840e0e5f7
Revert "YARN-3878. AsyncDispatcher can hang while stopping if it is configured for draining events on stop. (Varun Saxena via kasha)"
This reverts commit aa067c6aa47b4c79577096817acc00ad6421180c.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2466460d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2466460d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2466460d
Branch: refs/heads/HADOOP-12111
Commit: 2466460d4cd13ad5837c044476b26e63082c1d37
Parents: 19295b3
Author: Jian He <ji...@apache.org>
Authored: Mon Jul 13 14:30:35 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Jul 13 14:30:35 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 -
.../hadoop/yarn/event/AsyncDispatcher.java | 24 ++++----
.../hadoop/yarn/event/DrainDispatcher.java | 13 +----
.../hadoop/yarn/event/TestAsyncDispatcher.java | 61 --------------------
4 files changed, 14 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2466460d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 98f0e8d..5c17f04 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -648,9 +648,6 @@ Release 2.7.2 - UNRELEASED
YARN-3690. [JDK8] 'mvn site' fails. (Brahma Reddy Battula via aajisaka)
- YARN-3878. AsyncDispatcher can hang while stopping if it is configured for
- draining events on stop. (Varun Saxena via kasha)
-
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2466460d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 646611f..c54b9c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -55,6 +55,9 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
// stop functionality.
private volatile boolean drainEventsOnStop = false;
+ // Indicates all the remaining dispatcher's events on stop have been drained
+ // and processed.
+ private volatile boolean drained = true;
private Object waitForDrained = new Object();
// For drainEventsOnStop enabled only, block newly coming events into the
@@ -81,12 +84,13 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
@Override
public void run() {
while (!stopped && !Thread.currentThread().isInterrupted()) {
+ drained = eventQueue.isEmpty();
// blockNewEvents is only set when dispatcher is draining to stop,
// adding this check is to avoid the overhead of acquiring the lock
// and calling notify every time in the normal run of the loop.
if (blockNewEvents) {
synchronized (waitForDrained) {
- if (eventQueue.isEmpty()) {
+ if (drained) {
waitForDrained.notify();
}
}
@@ -135,7 +139,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
blockNewEvents = true;
LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
synchronized (waitForDrained) {
- while (!eventQueue.isEmpty() && eventHandlingThread.isAlive()) {
+ while (!drained && eventHandlingThread.isAlive()) {
waitForDrained.wait(1000);
LOG.info("Waiting for AsyncDispatcher to drain. Thread state is :" +
eventHandlingThread.getState());
@@ -219,21 +223,12 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
return handlerInstance;
}
- @VisibleForTesting
- protected boolean hasPendingEvents() {
- return !eventQueue.isEmpty();
- }
-
- @VisibleForTesting
- protected boolean isEventThreadWaiting() {
- return eventHandlingThread.getState() == Thread.State.WAITING;
- }
-
class GenericEventHandler implements EventHandler<Event> {
public void handle(Event event) {
if (blockNewEvents) {
return;
}
+ drained = false;
/* all this method does is enqueue all the events onto the queue */
int qSize = eventQueue.size();
@@ -290,4 +285,9 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
}
};
}
+
+ @VisibleForTesting
+ protected boolean isDrained() {
+ return this.drained;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2466460d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
index d1f4fe9..da5ae44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
@@ -27,24 +27,15 @@ public class DrainDispatcher extends AsyncDispatcher {
this(new LinkedBlockingQueue<Event>());
}
- public DrainDispatcher(BlockingQueue<Event> eventQueue) {
+ private DrainDispatcher(BlockingQueue<Event> eventQueue) {
super(eventQueue);
}
/**
- * Wait till event thread enters WAITING state (i.e. waiting for new events).
- */
- public void waitForEventThreadToWait() {
- while (!isEventThreadWaiting()) {
- Thread.yield();
- }
- }
-
- /**
* Busy loop waiting for all queued events to drain.
*/
public void await() {
- while (hasPendingEvents()) {
+ while (!isDrained()) {
Thread.yield();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2466460d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
deleted file mode 100644
index ee17ddd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.event;
-
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestAsyncDispatcher {
-
- /* This test checks whether dispatcher hangs on close if following two things
- * happen :
- * 1. A thread which was putting event to event queue is interrupted.
- * 2. Event queue is empty on close.
- */
- @SuppressWarnings({ "unchecked", "rawtypes" })
- @Test(timeout=10000)
- public void testDispatcherOnCloseIfQueueEmpty() throws Exception {
- BlockingQueue<Event> eventQueue = spy(new LinkedBlockingQueue<Event>());
- Event event = mock(Event.class);
- doThrow(new InterruptedException()).when(eventQueue).put(event);
- DrainDispatcher disp = new DrainDispatcher(eventQueue);
- disp.init(new Configuration());
- disp.setDrainEventsOnStop();
- disp.start();
- // Wait for event handler thread to start and begin waiting for events.
- disp.waitForEventThreadToWait();
- try {
- disp.getEventHandler().handle(event);
- } catch (YarnRuntimeException e) {
- }
- // Queue should be empty and dispatcher should not hang on close
- Assert.assertTrue("Event Queue should have been empty",
- eventQueue.isEmpty());
- disp.close();
- }
-}
[19/21] hadoop git commit: YARN-3805. Update the documentation of
Disk Checker based on YARN-90. Contributed by Masatake Iwasaki.
Posted by aw...@apache.org.
YARN-3805. Update the documentation of Disk Checker based on YARN-90. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ba2986d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ba2986d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ba2986d
Branch: refs/heads/HADOOP-12111
Commit: 1ba2986dee4bbb64d67ada005f8f132e69575274
Parents: 90bda9c
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Thu Jul 16 17:52:38 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Thu Jul 16 17:52:38 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop-yarn-site/src/site/markdown/NodeManager.md | 8 ++++++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba2986d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1e6c7d5..a098a64 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -634,6 +634,9 @@ Release 2.8.0 - UNRELEASED
YARN-3174. Consolidate the NodeManager and NodeManagerRestart documentation
into one. (Masatake Iwasaki via ozawa)
+ YARN-3805. Update the documentation of Disk Checker based on YARN-90.
+ (Masatake Iwasaki via ozawa)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba2986d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
index 69e99a7..4724ea6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
@@ -36,7 +36,9 @@ The NodeManager runs services to determine the health of the node it is executin
###Disk Checker
- The disk checker checks the state of the disks that the NodeManager is configured to use(local-dirs and log-dirs, configured using yarn.nodemanager.local-dirs and yarn.nodemanager.log-dirs respectively). The checks include permissions and free disk space. It also checks that the filesystem isn't in a read-only state. The checks are run at 2 minute intervals by default but can be configured to run as often as the user desires. If a disk fails the check, the NodeManager stops using that particular disk but still reports the node status as healthy. However if a number of disks fail the check(the number can be configured, as explained below), then the node is reported as unhealthy to the ResourceManager and new containers will not be assigned to the node. In addition, once a disk is marked as unhealthy, the NodeManager stops checking it to see if it has recovered(e.g. disk became full and was then cleaned up). The only way for the NodeManager to use that disk to restart the software o
n the node. The following configuration parameters can be used to modify the disk checks:
+ The disk checker checks the state of the disks that the NodeManager is configured to use(local-dirs and log-dirs, configured using yarn.nodemanager.local-dirs and yarn.nodemanager.log-dirs respectively). The checks include permissions and free disk space. It also checks that the filesystem isn't in a read-only state. The checks are run at 2 minute intervals by default but can be configured to run as often as the user desires. If a disk fails the check, the NodeManager stops using that particular disk but still reports the node status as healthy. However if a number of disks fail the check(the number can be configured, as explained below), then the node is reported as unhealthy to the ResourceManager and new containers will not be assigned to the node.
+
+The following configuration parameters can be used to modify the disk checks:
| Configuration Name | Allowed Values | Description |
|:---- |:---- |:---- |
@@ -48,7 +50,9 @@ The NodeManager runs services to determine the health of the node it is executin
###External Health Script
- Users may specify their own health checker script that will be invoked by the health checker service. Users may specify a timeout as well as options to be passed to the script. If the script exits with a non-zero exit code, times out or results in an exception being thrown, the node is marked as unhealthy. Please note that if the script cannot be executed due to permissions or an incorrect path, etc, then it counts as a failure and the node will be reported as unhealthy. Please note that speifying a health check script is not mandatory. If no script is specified, only the disk checker status will be used to determine the health of the node. The following configuration parameters can be used to set the health script:
+Users may specify their own health checker script that will be invoked by the health checker service. Users may specify a timeout as well as options to be passed to the script. If the script exits with a non-zero exit code, times out or results in an exception being thrown, the node is marked as unhealthy. Please note that if the script cannot be executed due to permissions or an incorrect path, etc, then it counts as a failure and the node will be reported as unhealthy. Please note that speifying a health check script is not mandatory. If no script is specified, only the disk checker status will be used to determine the health of the node.
+
+The following configuration parameters can be used to set the health script:
| Configuration Name | Allowed Values | Description |
|:---- |:---- |:---- |
[15/21] hadoop git commit: HDFS-8778.
TestBlockReportRateLimiting#testLeaseExpiration can deadlock. (Contributed by
Arpit Agarwal)
Posted by aw...@apache.org.
HDFS-8778. TestBlockReportRateLimiting#testLeaseExpiration can deadlock. (Contributed by Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ec0a044
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ec0a044
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ec0a044
Branch: refs/heads/HADOOP-12111
Commit: 3ec0a0444f75c8743289ec7c8645d4bdf51fc45a
Parents: edcaae4
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Jul 15 14:08:58 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Jul 15 14:08:58 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 5 +-
.../TestBlockReportRateLimiting.java | 64 ++++++--------------
2 files changed, 23 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ec0a044/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 20bdef0..8f6dd41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1044,6 +1044,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7608: hdfs dfsclient newConnectedPeer has no write timeout (Xiaoyu Yao
via Colin P. McCabe)
+ HDFS-8778. TestBlockReportRateLimiting#testLeaseExpiration can deadlock.
+ (Arpit Agarwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1059,7 +1062,7 @@ Release 2.7.2 - UNRELEASED
HDFS-8722. Optimize datanode writes for small writes and flushes (kihwal)
BUG FIXES
-
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ec0a044/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java
index fc5f9e7..86a7511 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java
@@ -24,7 +24,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FULL_BLOCK_REPOR
import com.google.common.base.Joiner;
import com.google.common.base.Supplier;
import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.commons.lang.mutable.MutableObject;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -42,8 +41,6 @@ import org.junit.Test;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -174,13 +171,11 @@ public class TestBlockReportRateLimiting {
conf.setLong(DFS_NAMENODE_FULL_BLOCK_REPORT_LEASE_LENGTH_MS, 100L);
final Semaphore gotFbrSem = new Semaphore(0);
- final AtomicReference<String> failure = new AtomicReference<String>("");
+ final AtomicReference<String> failure = new AtomicReference<>();
final AtomicReference<MiniDFSCluster> cluster =
- new AtomicReference<>(null);
- final BlockingQueue<Integer> datanodeToStop =
- new ArrayBlockingQueue<Integer>(1);
+ new AtomicReference<>();
+ final AtomicReference<String> datanodeToStop = new AtomicReference<>();
final BlockManagerFaultInjector injector = new BlockManagerFaultInjector() {
- private String uuidToStop = "";
@Override
public void incomingBlockReportRpc(DatanodeID nodeID,
@@ -189,11 +184,9 @@ public class TestBlockReportRateLimiting {
setFailure(failure, "Got unexpected rate-limiting-" +
"bypassing full block report RPC from " + nodeID);
}
- synchronized (this) {
- if (uuidToStop.equals(nodeID.getDatanodeUuid())) {
- throw new IOException("Injecting failure into block " +
- "report RPC for " + nodeID);
- }
+ if (nodeID.getXferAddr().equals(datanodeToStop.get())) {
+ throw new IOException("Injecting failure into block " +
+ "report RPC for " + nodeID);
}
gotFbrSem.release();
}
@@ -204,43 +197,24 @@ public class TestBlockReportRateLimiting {
if (leaseId == 0) {
return;
}
- synchronized (this) {
- if (uuidToStop.isEmpty()) {
- MiniDFSCluster cl;
- do {
- cl = cluster.get();
- } while (cl == null);
- int datanodeIndexToStop = getDatanodeIndex(cl, node);
- uuidToStop = node.getDatanodeUuid();
- datanodeToStop.add(Integer.valueOf(datanodeIndexToStop));
- }
- }
- }
-
- private int getDatanodeIndex(MiniDFSCluster cl,
- DatanodeDescriptor node) {
- List<DataNode> datanodes = cl.getDataNodes();
- for (int i = 0; i < datanodes.size(); i++) {
- DataNode datanode = datanodes.get(i);
- if (datanode.getDatanodeUuid().equals(node.getDatanodeUuid())) {
- return i;
- }
- }
- throw new RuntimeException("Failed to find UUID " +
- node.getDatanodeUuid() + " in the list of datanodes.");
+ datanodeToStop.compareAndSet(null, node.getXferAddr());
}
@Override
public void removeBlockReportLease(DatanodeDescriptor node, long leaseId) {
}
};
- BlockManagerFaultInjector.instance = injector;
- cluster.set(new MiniDFSCluster.Builder(conf).numDataNodes(2).build());
- cluster.get().waitActive();
- int datanodeIndexToStop = datanodeToStop.take();
- cluster.get().stopDataNode(datanodeIndexToStop);
- gotFbrSem.acquire();
- cluster.get().shutdown();
- Assert.assertEquals("", failure.get());
+ try {
+ BlockManagerFaultInjector.instance = injector;
+ cluster.set(new MiniDFSCluster.Builder(conf).numDataNodes(2).build());
+ cluster.get().waitActive();
+ Assert.assertNotNull(cluster.get().stopDataNode(datanodeToStop.get()));
+ gotFbrSem.acquire();
+ Assert.assertNull(failure.get());
+ } finally {
+ if (cluster.get() != null) {
+ cluster.get().shutdown();
+ }
+ }
}
}
[08/21] hadoop git commit: HDFS-7608: add CHANGES.txt
Posted by aw...@apache.org.
HDFS-7608: add CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7fb6ec4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7fb6ec4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7fb6ec4
Branch: refs/heads/HADOOP-12111
Commit: b7fb6ec4513de7d342c541eb3d9e14642286e2cf
Parents: 1d74cce
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jul 14 10:58:57 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jul 14 10:58:57 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7fb6ec4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e843dcc..86b1ea1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1038,6 +1038,9 @@ Release 2.8.0 - UNRELEASED
HDFS-2956. calling fetchdt without a --renewer argument throws NPE
(vinayakumarb)
+ HDFS-7608: hdfs dfsclient newConnectedPeer has no write timeout (Xiaoyu Yao
+ via Colin P. McCabe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
[07/21] hadoop git commit: HDFS-7608: hdfs dfsclient newConnectedPeer
has no write timeout (Xiaoyu Yao via Colin P. McCabe)
Posted by aw...@apache.org.
HDFS-7608: hdfs dfsclient newConnectedPeer has no write timeout (Xiaoyu Yao via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d74ccec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d74ccec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d74ccec
Branch: refs/heads/HADOOP-12111
Commit: 1d74ccececaefffaa90c0c18b40a3645dbc819d9
Parents: 4084eaf
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jul 14 10:57:59 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jul 14 10:57:59 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 1 +
.../hadoop/hdfs/TestDistributedFileSystem.java | 43 +++++++++++++++++---
2 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d74ccec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 4923a50..6629a83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3127,6 +3127,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
peer = TcpPeerServer.peerFromSocketAndKey(saslClient, sock, this,
blockToken, datanodeId);
peer.setReadTimeout(socketTimeout);
+ peer.setWriteTimeout(socketTimeout);
success = true;
return peer;
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d74ccec/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index f9da472..0b77210 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1132,10 +1132,9 @@ public class TestDistributedFileSystem {
cluster.shutdown();
}
}
-
-
+
@Test(timeout=10000)
- public void testDFSClientPeerTimeout() throws IOException {
+ public void testDFSClientPeerReadTimeout() throws IOException {
final int timeout = 1000;
final Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, timeout);
@@ -1152,11 +1151,11 @@ public class TestDistributedFileSystem {
long start = Time.now();
try {
peer.getInputStream().read();
- Assert.fail("should timeout");
+ Assert.fail("read should timeout");
} catch (SocketTimeoutException ste) {
long delta = Time.now() - start;
- Assert.assertTrue("timedout too soon", delta >= timeout*0.9);
- Assert.assertTrue("timedout too late", delta <= timeout*1.1);
+ Assert.assertTrue("read timedout too soon", delta >= timeout*0.9);
+ Assert.assertTrue("read timedout too late", delta <= timeout*1.1);
} catch (Throwable t) {
Assert.fail("wrong exception:"+t);
}
@@ -1178,4 +1177,36 @@ public class TestDistributedFileSystem {
cluster.shutdown();
}
}
+
+ @Test(timeout=10000)
+ public void testDFSClientPeerWriteTimeout() throws IOException {
+ final int timeout = 1000;
+ final Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, timeout);
+
+ // only need cluster to create a dfs client to get a peer
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ try {
+ cluster.waitActive();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ // Write 1 MB to a dummy socket to ensure the write times out
+ ServerSocket socket = new ServerSocket(0);
+ Peer peer = dfs.getClient().newConnectedPeer(
+ (InetSocketAddress) socket.getLocalSocketAddress(), null, null);
+ long start = Time.now();
+ try {
+ byte[] buf = new byte[1024 * 1024];
+ peer.getOutputStream().write(buf);
+ Assert.fail("write should timeout");
+ } catch (SocketTimeoutException ste) {
+ long delta = Time.now() - start;
+ Assert.assertTrue("write timedout too soon", delta >= timeout * 0.9);
+ Assert.assertTrue("write timedout too late", delta <= timeout * 1.1);
+ } catch (Throwable t) {
+ Assert.fail("wrong exception:" + t);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
[04/21] hadoop git commit: HADOOP-12211. Collect disk usage on the
node. Contributed by Robert Grandl
Posted by aw...@apache.org.
HADOOP-12211. Collect disk usage on the node. Contributed by Robert Grandl
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a431ed90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a431ed90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a431ed90
Branch: refs/heads/HADOOP-12111
Commit: a431ed9075cf6f467be5ff10f4ffb131cb1d3216
Parents: 9ef03a4
Author: Chris Douglas <cd...@apache.org>
Authored: Mon Jul 13 15:36:11 2015 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Mon Jul 13 15:36:11 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../java/org/apache/hadoop/util/SysInfo.java | 14 ++
.../org/apache/hadoop/util/SysInfoLinux.java | 157 ++++++++++++++++++-
.../org/apache/hadoop/util/SysInfoWindows.java | 12 ++
.../apache/hadoop/util/TestSysInfoLinux.java | 77 ++++++++-
.../gridmix/DummyResourceCalculatorPlugin.java | 17 ++
.../yarn/util/ResourceCalculatorPlugin.java | 18 +++
7 files changed, 293 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a431ed90/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3d4f1e4..3121430 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -695,6 +695,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12210. Collect network usage on the node (Robert Grandl via cdouglas)
+ HADOOP-12211. Collect disk usage on the node (Robert Grandl via cdouglas)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a431ed90/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
index 24b339d..b75a8d8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfo.java
@@ -120,4 +120,18 @@ public abstract class SysInfo {
*/
public abstract long getNetworkBytesWritten();
+ /**
+ * Obtain the aggregated number of bytes read from disks.
+ *
+ * @return total number of bytes read.
+ */
+ public abstract long getStorageBytesRead();
+
+ /**
+ * Obtain the aggregated number of bytes written to disks.
+ *
+ * @return total number of bytes written.
+ */
+ public abstract long getStorageBytesWritten();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a431ed90/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 8801985..6b21f18 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -25,6 +25,7 @@ import java.io.InputStreamReader;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.charset.Charset;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -94,11 +95,27 @@ public class SysInfoLinux extends SysInfo {
"[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
"[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+).*");
+ /**
+ * Pattern for parsing /proc/diskstats.
+ */
+ private static final String PROCFS_DISKSFILE = "/proc/diskstats";
+ private static final Pattern PROCFS_DISKSFILE_FORMAT =
+ Pattern.compile("^[ \t]*([0-9]+)[ \t]*([0-9 ]+)" +
+ "(?!([a-zA-Z]+[0-9]+))([a-zA-Z]+)" +
+ "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
+ "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
+ "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)");
+ /**
+ * Pattern for parsing /sys/block/partition_name/queue/hw_sector_size.
+ */
+ private static final Pattern PROCFS_DISKSECTORFILE_FORMAT =
+ Pattern.compile("^([0-9]+)");
private String procfsMemFile;
private String procfsCpuFile;
private String procfsStatFile;
private String procfsNetFile;
+ private String procfsDisksFile;
private long jiffyLengthInMillis;
private long ramSize = 0;
@@ -113,10 +130,15 @@ public class SysInfoLinux extends SysInfo {
private long cpuFrequency = 0L; // CPU frequency on the system (kHz)
private long numNetBytesRead = 0L; // aggregated bytes read from network
private long numNetBytesWritten = 0L; // aggregated bytes written to network
+ private long numDisksBytesRead = 0L; // aggregated bytes read from disks
+ private long numDisksBytesWritten = 0L; // aggregated bytes written to disks
private boolean readMemInfoFile = false;
private boolean readCpuInfoFile = false;
+ /* map for every disk its sector size */
+ private HashMap<String, Integer> perDiskSectorSize = null;
+
public static final long PAGE_SIZE = getConf("PAGESIZE");
public static final long JIFFY_LENGTH_IN_MILLIS =
Math.max(Math.round(1000D / getConf("CLK_TCK")), -1);
@@ -145,7 +167,7 @@ public class SysInfoLinux extends SysInfo {
public SysInfoLinux() {
this(PROCFS_MEMFILE, PROCFS_CPUINFO, PROCFS_STAT,
- PROCFS_NETFILE, JIFFY_LENGTH_IN_MILLIS);
+ PROCFS_NETFILE, PROCFS_DISKSFILE, JIFFY_LENGTH_IN_MILLIS);
}
/**
@@ -155,6 +177,7 @@ public class SysInfoLinux extends SysInfo {
* @param procfsCpuFile fake file for /proc/cpuinfo
* @param procfsStatFile fake file for /proc/stat
* @param procfsNetFile fake file for /proc/net/dev
+ * @param procfsDisksFile fake file for /proc/diskstats
* @param jiffyLengthInMillis fake jiffy length value
*/
@VisibleForTesting
@@ -162,13 +185,16 @@ public class SysInfoLinux extends SysInfo {
String procfsCpuFile,
String procfsStatFile,
String procfsNetFile,
+ String procfsDisksFile,
long jiffyLengthInMillis) {
this.procfsMemFile = procfsMemFile;
this.procfsCpuFile = procfsCpuFile;
this.procfsStatFile = procfsStatFile;
this.procfsNetFile = procfsNetFile;
+ this.procfsDisksFile = procfsDisksFile;
this.jiffyLengthInMillis = jiffyLengthInMillis;
this.cpuTimeTracker = new CpuTimeTracker(jiffyLengthInMillis);
+ this.perDiskSectorSize = new HashMap<String, Integer>();
}
/**
@@ -411,6 +437,119 @@ public class SysInfoLinux extends SysInfo {
}
}
+ /**
+ * Read /proc/diskstats file, parse and calculate amount
+ * of bytes read and written from/to disks.
+ */
+ private void readProcDisksInfoFile() {
+
+ numDisksBytesRead = 0L;
+ numDisksBytesWritten = 0L;
+
+ // Read "/proc/diskstats" file
+ BufferedReader in;
+ try {
+ in = new BufferedReader(new InputStreamReader(
+ new FileInputStream(procfsDisksFile), Charset.forName("UTF-8")));
+ } catch (FileNotFoundException f) {
+ return;
+ }
+
+ Matcher mat;
+ try {
+ String str = in.readLine();
+ while (str != null) {
+ mat = PROCFS_DISKSFILE_FORMAT.matcher(str);
+ if (mat.find()) {
+ String diskName = mat.group(4);
+ assert diskName != null;
+ // ignore loop or ram partitions
+ if (diskName.contains("loop") || diskName.contains("ram")) {
+ str = in.readLine();
+ continue;
+ }
+
+ Integer sectorSize;
+ synchronized (perDiskSectorSize) {
+ sectorSize = perDiskSectorSize.get(diskName);
+ if (null == sectorSize) {
+ // retrieve sectorSize
+ // if unavailable or error, assume 512
+ sectorSize = readDiskBlockInformation(diskName, 512);
+ perDiskSectorSize.put(diskName, sectorSize);
+ }
+ }
+
+ String sectorsRead = mat.group(7);
+ String sectorsWritten = mat.group(11);
+ if (null == sectorsRead || null == sectorsWritten) {
+ return;
+ }
+ numDisksBytesRead += Long.parseLong(sectorsRead) * sectorSize;
+ numDisksBytesWritten += Long.parseLong(sectorsWritten) * sectorSize;
+ }
+ str = in.readLine();
+ }
+ } catch (IOException e) {
+ LOG.warn("Error reading the stream " + procfsDisksFile, e);
+ } finally {
+ // Close the streams
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.warn("Error closing the stream " + procfsDisksFile, e);
+ }
+ }
+ }
+
+ /**
+ * Read /sys/block/diskName/queue/hw_sector_size file, parse and calculate
+ * sector size for a specific disk.
+ * @return sector size of specified disk, or defSector
+ */
+ int readDiskBlockInformation(String diskName, int defSector) {
+
+ assert perDiskSectorSize != null && diskName != null;
+
+ String procfsDiskSectorFile =
+ "/sys/block/" + diskName + "/queue/hw_sector_size";
+
+ BufferedReader in;
+ try {
+ in = new BufferedReader(new InputStreamReader(
+ new FileInputStream(procfsDiskSectorFile),
+ Charset.forName("UTF-8")));
+ } catch (FileNotFoundException f) {
+ return defSector;
+ }
+
+ Matcher mat;
+ try {
+ String str = in.readLine();
+ while (str != null) {
+ mat = PROCFS_DISKSECTORFILE_FORMAT.matcher(str);
+ if (mat.find()) {
+ String secSize = mat.group(1);
+ if (secSize != null) {
+ return Integer.parseInt(secSize);
+ }
+ }
+ str = in.readLine();
+ }
+ return defSector;
+ } catch (IOException|NumberFormatException e) {
+ LOG.warn("Error reading the stream " + procfsDiskSectorFile, e);
+ return defSector;
+ } finally {
+ // Close the streams
+ try {
+ in.close();
+ } catch (IOException e) {
+ LOG.warn("Error closing the stream " + procfsDiskSectorFile, e);
+ }
+ }
+ }
+
/** {@inheritDoc} */
@Override
public long getPhysicalMemorySize() {
@@ -492,6 +631,18 @@ public class SysInfoLinux extends SysInfo {
return numNetBytesWritten;
}
+ @Override
+ public long getStorageBytesRead() {
+ readProcDisksInfoFile();
+ return numDisksBytesRead;
+ }
+
+ @Override
+ public long getStorageBytesWritten() {
+ readProcDisksInfoFile();
+ return numDisksBytesWritten;
+ }
+
/**
* Test the {@link SysInfoLinux}.
*
@@ -515,6 +666,10 @@ public class SysInfoLinux extends SysInfo {
+ plugin.getNetworkBytesRead());
System.out.println("Total network written (bytes) : "
+ plugin.getNetworkBytesWritten());
+ System.out.println("Total storage read (bytes) : "
+ + plugin.getStorageBytesRead());
+ System.out.println("Total storage written (bytes) : "
+ + plugin.getStorageBytesWritten());
try {
// Sleep so we can compute the CPU usage
Thread.sleep(500L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a431ed90/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
index f8542a3..f3fb364 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoWindows.java
@@ -193,4 +193,16 @@ public class SysInfoWindows extends SysInfo {
return 0L;
}
+ @Override
+ public long getStorageBytesRead() {
+ // TODO unimplemented
+ return 0L;
+ }
+
+ @Override
+ public long getStorageBytesWritten() {
+ // TODO unimplemented
+ return 0L;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a431ed90/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index 2a31f31..83f2e86 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -37,17 +37,18 @@ public class TestSysInfoLinux {
/**
* LinuxResourceCalculatorPlugin with a fake timer
*/
- static class FakeLinuxResourceCalculatorPlugin extends
- SysInfoLinux {
+ static class FakeLinuxResourceCalculatorPlugin extends SysInfoLinux {
+ static final int SECTORSIZE = 4096;
long currentTime = 0;
public FakeLinuxResourceCalculatorPlugin(String procfsMemFile,
String procfsCpuFile,
String procfsStatFile,
String procfsNetFile,
+ String procfsDisksFile,
long jiffyLengthInMillis) {
super(procfsMemFile, procfsCpuFile, procfsStatFile, procfsNetFile,
- jiffyLengthInMillis);
+ procfsDisksFile, jiffyLengthInMillis);
}
@Override
long getCurrentTime() {
@@ -56,6 +57,10 @@ public class TestSysInfoLinux {
public void advanceTime(long adv) {
currentTime += adv * this.getJiffyLengthInMillis();
}
+ @Override
+ int readDiskBlockInformation(String diskName, int defSector) {
+ return SECTORSIZE;
+ }
}
private static final FakeLinuxResourceCalculatorPlugin plugin;
private static String TEST_ROOT_DIR = new Path(System.getProperty(
@@ -64,6 +69,7 @@ public class TestSysInfoLinux {
private static final String FAKE_CPUFILE;
private static final String FAKE_STATFILE;
private static final String FAKE_NETFILE;
+ private static final String FAKE_DISKSFILE;
private static final long FAKE_JIFFY_LENGTH = 10L;
static {
int randomNum = (new Random()).nextInt(1000000000);
@@ -71,9 +77,11 @@ public class TestSysInfoLinux {
FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum;
FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum;
FAKE_NETFILE = TEST_ROOT_DIR + File.separator + "NETINFO_" + randomNum;
+ FAKE_DISKSFILE = TEST_ROOT_DIR + File.separator + "DISKSINFO_" + randomNum;
plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE,
FAKE_STATFILE,
FAKE_NETFILE,
+ FAKE_DISKSFILE,
FAKE_JIFFY_LENGTH);
}
static final String MEMINFO_FORMAT =
@@ -157,6 +165,38 @@ public class TestSysInfoLinux {
" eth1: %d 3152521 0 0 0 0 0 219781 %d 1866290 0 0 " +
"0 0 0 0\n";
+ static final String DISKSINFO_FORMAT =
+ "1 0 ram0 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "1 1 ram1 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "1 2 ram2 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "1 3 ram3 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "1 4 ram4 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "1 5 ram5 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "1 6 ram6 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "7 0 loop0 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "7 1 loop1 0 0 0 0 0 0 0 0 0 0 0\n"+
+ "8 0 sda 82575678 2486518 %d 59876600 3225402 19761924 %d " +
+ "6407705 4 48803346 66227952\n"+
+ "8 1 sda1 732 289 21354 787 7 3 32 4 0 769 791"+
+ "8 2 sda2 744272 2206315 23605200 6742762 336830 2979630 " +
+ "26539520 1424776 4 1820130 8165444\n"+
+ "8 3 sda3 81830497 279914 17881852954 53132969 2888558 16782291 " +
+ "157367552 4982925 0 47077660 58061635\n"+
+ "8 32 sdc 10148118 693255 %d 122125461 6090515 401630172 %d 2696685590 " +
+ "0 26848216 2818793840\n"+
+ "8 33 sdc1 10147917 693230 2054138426 122125426 6090506 401630172 " +
+ "3261765880 2696685589 0 26848181 2818793804\n"+
+ "8 64 sde 9989771 553047 %d 93407551 5978572 391997273 %d 2388274325 " +
+ "0 24396646 2481664818\n"+
+ "8 65 sde1 9989570 553022 1943973346 93407489 5978563 391997273 3183807264 " +
+ "2388274325 0 24396584 2481666274\n"+
+ "8 80 sdf 10197163 693995 %d 144374395 6216644 408395438 %d 2669389056 0 " +
+ "26164759 2813746348\n"+
+ "8 81 sdf1 10196962 693970 2033452794 144374355 6216635 408395438 3316897064 " +
+ "2669389056 0 26164719 2813746308\n"+
+ "8 129 sdi1 10078602 657936 2056552626 108362198 6134036 403851153 3279882064 " +
+ "2639256086 0 26260432 2747601085\n";
+
/**
* Test parsing /proc/stat and /proc/cpuinfo
* @throws IOException
@@ -358,4 +398,35 @@ public class TestSysInfoLinux {
assertEquals(plugin.getNetworkBytesWritten(), numBytesWrittenIntf1 + numBytesWrittenIntf2);
}
+ /**
+ * Test parsing /proc/diskstats
+ * @throws IOException
+ */
+ @Test
+ public void parsingProcDisksFile() throws IOException {
+ long numSectorsReadsda = 1790549L; long numSectorsWrittensda = 1839071L;
+ long numSectorsReadsdc = 20541402L; long numSectorsWrittensdc = 32617658L;
+ long numSectorsReadsde = 19439751L; long numSectorsWrittensde = 31838072L;
+ long numSectorsReadsdf = 20334546L; long numSectorsWrittensdf = 33168970L;
+ File tempFile = new File(FAKE_DISKSFILE);
+ tempFile.deleteOnExit();
+ FileWriter fWriter = new FileWriter(FAKE_DISKSFILE);
+ fWriter.write(String.format(DISKSINFO_FORMAT,
+ numSectorsReadsda, numSectorsWrittensda,
+ numSectorsReadsdc, numSectorsWrittensdc,
+ numSectorsReadsde, numSectorsWrittensde,
+ numSectorsReadsdf, numSectorsWrittensdf));
+
+ fWriter.close();
+ long expectedNumSectorsRead = numSectorsReadsda + numSectorsReadsdc +
+ numSectorsReadsde + numSectorsReadsdf;
+ long expectedNumSectorsWritten = numSectorsWrittensda + numSectorsWrittensdc +
+ numSectorsWrittensde + numSectorsWrittensdf;
+ // use non-default sector size
+ int diskSectorSize = FakeLinuxResourceCalculatorPlugin.SECTORSIZE;
+ assertEquals(expectedNumSectorsRead * diskSectorSize,
+ plugin.getStorageBytesRead());
+ assertEquals(expectedNumSectorsWritten * diskSectorSize,
+ plugin.getStorageBytesWritten());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a431ed90/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
index b86303b..4999f14 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java
@@ -54,6 +54,12 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
/** cumulative number of bytes written over the network */
public static final String NETWORK_BYTES_WRITTEN =
"mapred.tasktracker.networkwritten.testing";
+ /** cumulative number of bytes read from disks */
+ public static final String STORAGE_BYTES_READ =
+ "mapred.tasktracker.storageread.testing";
+ /** cumulative number of bytes written to disks */
+ public static final String STORAGE_BYTES_WRITTEN =
+ "mapred.tasktracker.storagewritten.testing";
/** process cumulative CPU usage time for testing */
public static final String PROC_CUMULATIVE_CPU_TIME =
"mapred.tasktracker.proccumulativecputime.testing";
@@ -130,4 +136,15 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
return getConf().getLong(NETWORK_BYTES_WRITTEN, -1);
}
+ /** {@inheritDoc} */
+ @Override
+ public long getStorageBytesRead() {
+ return getConf().getLong(STORAGE_BYTES_READ, -1);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getStorageBytesWritten() {
+ return getConf().getLong(STORAGE_BYTES_WRITTEN, -1);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a431ed90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
index 3af4aee..691c4ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
@@ -145,6 +145,24 @@ public class ResourceCalculatorPlugin extends Configured {
}
/**
+ * Obtain the aggregated number of bytes read from disks.
+ *
+ * @return total number of bytes read.
+ */
+ public long getStorageBytesRead() {
+ return sys.getStorageBytesRead();
+ }
+
+ /**
+ * Obtain the aggregated number of bytes written to disks.
+ *
+ * @return total number of bytes written.
+ */
+ public long getStorageBytesWritten() {
+ return sys.getStorageBytesWritten();
+ }
+
+ /**
* Create the ResourceCalculatorPlugin from the class name and configure it. If
* class name is null, this method will try and return a memory calculator
* plugin available for this system.
[13/21] hadoop git commit: HADOOP-12153. ByteBufferReadable doesn't
declare @InterfaceAudience and @InterfaceStability. Contributed by Brahma
Reddy Battula.
Posted by aw...@apache.org.
HADOOP-12153. ByteBufferReadable doesn't declare @InterfaceAudience and @InterfaceStability. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cec1d43d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cec1d43d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cec1d43d
Branch: refs/heads/HADOOP-12111
Commit: cec1d43db026e66a9e84b5c3e8476dfd33f17ecb
Parents: 0a16ee6
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Jul 15 14:18:12 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Wed Jul 15 14:19:17 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java | 4 ++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cec1d43d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a807d12..3d64156 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -697,6 +697,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12211. Collect disk usage on the node (Robert Grandl via cdouglas)
+ HADOOP-12153. ByteBufferReadable doesn't declare @InterfaceAudience and
+ @InterfaceStability. (Brahma Reddy Battula via ozawa)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cec1d43d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
index aa6e85e..20f7224 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
@@ -19,11 +19,15 @@ package org.apache.hadoop.fs;
import java.io.IOException;
import java.nio.ByteBuffer;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
/**
* Implementers of this interface provide a read API that writes to a
* ByteBuffer, not a byte[].
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public interface ByteBufferReadable {
/**
* Reads up to buf.remaining() bytes into buf. Callers should use
[09/21] hadoop git commit: HDFS-8722. Optimize datanode writes for
small writes and flushes. Contributed by Kihwal Lee
Posted by aw...@apache.org.
HDFS-8722. Optimize datanode writes for small writes and flushes. Contributed by Kihwal Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59388a80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59388a80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59388a80
Branch: refs/heads/HADOOP-12111
Commit: 59388a801514d6af64ef27fbf246d8054f1dcc74
Parents: b7fb6ec
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Jul 14 14:04:06 2015 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Jul 14 14:04:06 2015 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hdfs/server/datanode/BlockReceiver.java | 34 +++++++++++++-------
2 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59388a80/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 86b1ea1..14f3403 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1053,6 +1053,8 @@ Release 2.7.2 - UNRELEASED
OPTIMIZATIONS
+ HDFS-8722. Optimize datanode writes for small writes and flushes (kihwal)
+
BUG FIXES
Release 2.7.1 - 2015-07-06
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59388a80/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 2468f43..55c9d57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -598,14 +598,19 @@ class BlockReceiver implements Closeable {
// bytes should be skipped when writing the data and checksum
// buffers out to disk.
long partialChunkSizeOnDisk = onDiskLen % bytesPerChecksum;
+ long lastChunkBoundary = onDiskLen - partialChunkSizeOnDisk;
boolean alignedOnDisk = partialChunkSizeOnDisk == 0;
boolean alignedInPacket = firstByteInBlock % bytesPerChecksum == 0;
- // Since data is always appended, not overwritten, partial CRC
- // recalculation is necessary if the on-disk data is not chunk-
- // aligned, regardless of whether the beginning of the data in
- // the packet is chunk-aligned.
- boolean doPartialCrc = !alignedOnDisk && !shouldNotWriteChecksum;
+ // If the end of the on-disk data is not chunk-aligned, the last
+ // checksum needs to be overwritten.
+ boolean overwriteLastCrc = !alignedOnDisk && !shouldNotWriteChecksum;
+ // If the starting offset of the packat data is at the last chunk
+ // boundary of the data on disk, the partial checksum recalculation
+ // can be skipped and the checksum supplied by the client can be used
+ // instead. This reduces disk reads and cpu load.
+ boolean doCrcRecalc = overwriteLastCrc &&
+ (lastChunkBoundary != firstByteInBlock);
// If this is a partial chunk, then verify that this is the only
// chunk in the packet. If the starting offset is not chunk
@@ -621,9 +626,10 @@ class BlockReceiver implements Closeable {
// If the last portion of the block file is not a full chunk,
// then read in pre-existing partial data chunk and recalculate
// the checksum so that the checksum calculation can continue
- // from the right state.
+ // from the right state. If the client provided the checksum for
+ // the whole chunk, this is not necessary.
Checksum partialCrc = null;
- if (doPartialCrc) {
+ if (doCrcRecalc) {
if (LOG.isDebugEnabled()) {
LOG.debug("receivePacket for " + block
+ ": previous write did not end at the chunk boundary."
@@ -659,8 +665,15 @@ class BlockReceiver implements Closeable {
int skip = 0;
byte[] crcBytes = null;
- // First, overwrite the partial crc at the end, if necessary.
- if (doPartialCrc) { // not chunk-aligned on disk
+ // First, prepare to overwrite the partial crc at the end.
+ if (overwriteLastCrc) { // not chunk-aligned on disk
+ // prepare to overwrite last checksum
+ adjustCrcFilePosition();
+ }
+
+ // The CRC was recalculated for the last partial chunk. Update the
+ // CRC by reading the rest of the chunk, then write it out.
+ if (doCrcRecalc) {
// Calculate new crc for this chunk.
int bytesToReadForRecalc =
(int)(bytesPerChecksum - partialChunkSizeOnDisk);
@@ -673,8 +686,6 @@ class BlockReceiver implements Closeable {
byte[] buf = FSOutputSummer.convertToByteStream(partialCrc,
checksumSize);
crcBytes = copyLastChunkChecksum(buf, checksumSize, buf.length);
- // prepare to overwrite last checksum
- adjustCrcFilePosition();
checksumOut.write(buf);
if(LOG.isDebugEnabled()) {
LOG.debug("Writing out partial crc for data len " + len +
@@ -687,7 +698,6 @@ class BlockReceiver implements Closeable {
// boundary. The checksum after the boundary was already counted
// above. Only count the number of checksums skipped up to the
// boundary here.
- long lastChunkBoundary = onDiskLen - (onDiskLen%bytesPerChecksum);
long skippedDataBytes = lastChunkBoundary - firstByteInBlock;
if (skippedDataBytes > 0) {
[06/21] hadoop git commit: HADOOP-12191. Bzip2Factory is not thread
safe. Contributed by Brahma Reddy Battula.
Posted by aw...@apache.org.
HADOOP-12191. Bzip2Factory is not thread safe. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4084eaf9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4084eaf9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4084eaf9
Branch: refs/heads/HADOOP-12111
Commit: 4084eaf94347042d9b8cb1e86ba831036621dfdd
Parents: ac94ba3
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Jul 14 19:13:45 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Jul 14 19:14:42 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4084eaf9/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3121430..a9bd7de 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -973,6 +973,9 @@ Release 2.7.2 - UNRELEASED
HADOOP-12186. ActiveStandbyElector shouldn't call monitorLockNodeAsync
multiple times (zhihai xu via vinayakumarb)
+ HADOOP-12191. Bzip2Factory is not thread safe. (Brahma Reddy Battula
+ via ozawa)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4084eaf9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
index 32fe910..45f1edd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
@@ -44,7 +44,7 @@ public class Bzip2Factory {
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
- public static boolean isNativeBzip2Loaded(Configuration conf) {
+ public static synchronized boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
[10/21] hadoop git commit: HDFS-8742. Inotify: Support event for
OP_TRUNCATE. Contributed by Surendra Singh Lilhore.
Posted by aw...@apache.org.
HDFS-8742. Inotify: Support event for OP_TRUNCATE. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/979c9ca2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/979c9ca2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/979c9ca2
Branch: refs/heads/HADOOP-12111
Commit: 979c9ca2ca89e99dc7165abfa29c78d66de43d9a
Parents: 59388a8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Jul 15 04:41:54 2015 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Jul 15 04:42:08 2015 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/inotify/Event.java | 37 +++++++++++++++++++-
.../src/main/proto/inotify.proto | 7 ++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 17 +++++++++
.../namenode/InotifyFSEditLogOpTranslator.java | 4 +++
.../hdfs/TestDFSInotifyEventInputStream.java | 17 ++++++++-
6 files changed, 83 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979c9ca2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index 53eefa0..dee17a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -35,7 +35,7 @@ import java.util.List;
@InterfaceStability.Unstable
public abstract class Event {
public static enum EventType {
- CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK
+ CREATE, CLOSE, APPEND, RENAME, METADATA, UNLINK, TRUNCATE
}
private EventType eventType;
@@ -542,4 +542,39 @@ public abstract class Event {
return timestamp;
}
}
+
+ /**
+ * Sent when a file is truncated.
+ */
+ public static class TruncateEvent extends Event {
+ private String path;
+ private long fileSize;
+ private long timestamp;
+
+
+ public TruncateEvent(String path, long fileSize, long timestamp) {
+ super(EventType.TRUNCATE);
+ this.path = path;
+ this.fileSize = fileSize;
+ this.timestamp = timestamp;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ /**
+ * The size of the truncated file in bytes.
+ */
+ public long getFileSize() {
+ return fileSize;
+ }
+
+ /**
+ * The time when this event occurred, in milliseconds since the epoch.
+ */
+ public long getTimestamp() {
+ return timestamp;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979c9ca2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
index 5b78fe6..5339902 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
@@ -41,6 +41,7 @@ enum EventType {
EVENT_RENAME = 0x3;
EVENT_METADATA = 0x4;
EVENT_UNLINK = 0x5;
+ EVENT_TRUNCATE = 0x6;
}
message EventProto {
@@ -87,6 +88,12 @@ message CloseEventProto {
required int64 timestamp = 3;
}
+message TruncateEventProto {
+ required string path = 1;
+ required int64 fileSize = 2;
+ required int64 timestamp = 3;
+}
+
message AppendEventProto {
required string path = 1;
optional bool newBlock = 2 [default = false];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979c9ca2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 14f3403..20bdef0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -719,6 +719,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move
progress. (Surendra Singh Lilhore via szetszwo)
+ HDFS-8742. Inotify: Support event for OP_TRUNCATE.
+ (Surendra Singh Lilhore via aajisaka)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979c9ca2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 32d9614..4ca5b26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -2685,6 +2685,12 @@ public class PBHelper {
.timestamp(unlink.getTimestamp())
.build());
break;
+ case EVENT_TRUNCATE:
+ InotifyProtos.TruncateEventProto truncate =
+ InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
+ events.add(new Event.TruncateEvent(truncate.getPath(),
+ truncate.getFileSize(), truncate.getTimestamp()));
+ break;
default:
throw new RuntimeException("Unexpected inotify event type: " +
p.getType());
@@ -2791,6 +2797,17 @@ public class PBHelper {
.setTimestamp(ue.getTimestamp()).build().toByteString()
).build());
break;
+ case TRUNCATE:
+ Event.TruncateEvent te = (Event.TruncateEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_TRUNCATE)
+ .setContents(
+ InotifyProtos.TruncateEventProto.newBuilder()
+ .setPath(te.getPath())
+ .setFileSize(te.getFileSize())
+ .setTimestamp(te.getTimestamp()).build().toByteString()
+ ).build());
+ break;
default:
throw new RuntimeException("Unexpected inotify event: " + e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979c9ca2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
index 5345b46..0918107 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
@@ -176,6 +176,10 @@ public class InotifyFSEditLogOpTranslator {
.metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS)
.path(saOp.src)
.acls(saOp.aclEntries).build() });
+ case OP_TRUNCATE:
+ FSEditLogOp.TruncateOp tOp = (FSEditLogOp.TruncateOp) op;
+ return new EventBatch(op.txid, new Event[] {
+ new Event.TruncateEvent(tOp.src, tOp.newLength, tOp.timestamp) });
default:
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979c9ca2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index ba33bd3..385d653 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -102,6 +102,8 @@ public class TestDFSInotifyEventInputStream {
DFSTestUtil.createFile(fs, new Path("/file"), BLOCK_SIZE, (short) 1, 0L);
DFSTestUtil.createFile(fs, new Path("/file3"), BLOCK_SIZE, (short) 1, 0L);
DFSTestUtil.createFile(fs, new Path("/file5"), BLOCK_SIZE, (short) 1, 0L);
+ DFSTestUtil.createFile(fs, new Path("/truncate_file"),
+ BLOCK_SIZE * 2, (short) 1, 0L);
DFSInotifyEventInputStream eis = client.getInotifyEventStream();
client.rename("/file", "/file4", null); // RenameOp -> RenameEvent
client.rename("/file4", "/file2"); // RenameOldOp -> RenameEvent
@@ -136,7 +138,8 @@ public class TestDFSInotifyEventInputStream {
"user::rwx,user:foo:rw-,group::r--,other::---", true));
client.removeAcl("/file5"); // SetAclOp -> MetadataUpdateEvent
client.rename("/file5", "/dir"); // RenameOldOp -> RenameEvent
-
+ //TruncateOp -> TruncateEvent
+ client.truncate("/truncate_file", BLOCK_SIZE);
EventBatch batch = null;
// RenameOp
@@ -354,6 +357,18 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(re3.getSrcPath().equals("/file5"));
Assert.assertTrue(re.getTimestamp() > 0);
+ // TruncateOp
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert
+ .assertTrue(batch.getEvents()[0].getEventType() ==
+ Event.EventType.TRUNCATE);
+ Event.TruncateEvent et = ((Event.TruncateEvent) batch.getEvents()[0]);
+ Assert.assertTrue(et.getPath().equals("/truncate_file"));
+ Assert.assertTrue(et.getFileSize() == BLOCK_SIZE);
+ Assert.assertTrue(et.getTimestamp() > 0);
+
// Returns null when there are no further events
Assert.assertTrue(eis.poll() == null);
[14/21] hadoop git commit: YARN-3170. YARN architecture document
needs updating. Contirubted by Brahma Reddy Battula.
Posted by aw...@apache.org.
YARN-3170. YARN architecture document needs updating. Contirubted by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edcaae44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edcaae44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edcaae44
Branch: refs/heads/HADOOP-12111
Commit: edcaae44c10b7e88e68fa97afd32e4da4a9d8df7
Parents: cec1d43
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Jul 15 15:42:41 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Wed Jul 15 15:42:41 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop-yarn-site/src/site/markdown/YARN.md | 22 +++++++-------------
2 files changed, 10 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcaae44/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 780c667..0a6f871 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -639,6 +639,9 @@ Release 2.7.2 - UNRELEASED
IMPROVEMENTS
+ YARN-3170. YARN architecture document needs updating. (Brahma Reddy Battula
+ via ozawa)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcaae44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
index f79272c..f8e8154 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
@@ -12,14 +12,12 @@
limitations under the License. See accompanying LICENSE file.
-->
-Apache Hadoop NextGen MapReduce (YARN)
+Apache Hadoop YARN
==================
-MapReduce has undergone a complete overhaul in hadoop-0.23 and we now have, what we call, MapReduce 2.0 (MRv2) or YARN.
+The fundamental idea of YARN is to split up the functionalities of resource management and job scheduling/monitoring into separate daemons. The idea is to have a global ResourceManager (*RM*) and per-application ApplicationMaster (*AM*). An application is either a single job or a DAG of jobs.
-The fundamental idea of MRv2 is to split up the two major functionalities of the JobTracker, resource management and job scheduling/monitoring, into separate daemons. The idea is to have a global ResourceManager (*RM*) and per-application ApplicationMaster (*AM*). An application is either a single job in the classical sense of Map-Reduce jobs or a DAG of jobs.
-
-The ResourceManager and per-node slave, the NodeManager (*NM*), form the data-computation framework. The ResourceManager is the ultimate authority that arbitrates resources among all the applications in the system.
+The ResourceManager and the NodeManager form the data-computation framework. The ResourceManager is the ultimate authority that arbitrates resources among all the applications in the system. The NodeManager is the per-machine framework agent who is responsible for containers, monitoring their resource usage (cpu, memory, disk, network) and reporting the same to the ResourceManager/Scheduler.
The per-application ApplicationMaster is, in effect, a framework specific library and is tasked with negotiating resources from the ResourceManager and working with the NodeManager(s) to execute and monitor the tasks.
@@ -27,16 +25,10 @@ The per-application ApplicationMaster is, in effect, a framework specific librar
The ResourceManager has two main components: Scheduler and ApplicationsManager.
-The Scheduler is responsible for allocating resources to the various running applications subject to familiar constraints of capacities, queues etc. The Scheduler is pure scheduler in the sense that it performs no monitoring or tracking of status for the application. Also, it offers no guarantees about restarting failed tasks either due to application failure or hardware failures. The Scheduler performs its scheduling function based the resource requirements of the applications; it does so based on the abstract notion of a resource *Container* which incorporates elements such as memory, cpu, disk, network etc. In the first version, only `memory` is supported.
-
-The Scheduler has a pluggable policy plug-in, which is responsible for partitioning the cluster resources among the various queues, applications etc. The current Map-Reduce schedulers such as the CapacityScheduler and the FairScheduler would be some examples of the plug-in.
-
-The CapacityScheduler supports `hierarchical queues` to allow for more predictable sharing of cluster resources
-
-The ApplicationsManager is responsible for accepting job-submissions, negotiating the first container for executing the application specific ApplicationMaster and provides the service for restarting the ApplicationMaster container on failure.
+The Scheduler is responsible for allocating resources to the various running applications subject to familiar constraints of capacities, queues etc. The Scheduler is pure scheduler in the sense that it performs no monitoring or tracking of status for the application. Also, it offers no guarantees about restarting failed tasks either due to application failure or hardware failures. The Scheduler performs its scheduling function based the resource requirements of the applications; it does so based on the abstract notion of a resource *Container* which incorporates elements such as memory, cpu, disk, network etc.
-The NodeManager is the per-machine framework agent who is responsible for containers, monitoring their resource usage (cpu, memory, disk, network) and reporting the same to the ResourceManager/Scheduler.
+The Scheduler has a pluggable policy which is responsible for partitioning the cluster resources among the various queues, applications etc. The current schedulers such as the [CapacityScheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html) and the [FairScheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/FairScheduler.html) would be some examples of plug-ins.
-The per-application ApplicationMaster has the responsibility of negotiating appropriate resource containers from the Scheduler, tracking their status and monitoring for progress.
+The ApplicationsManager is responsible for accepting job-submissions, negotiating the first container for executing the application specific ApplicationMaster and provides the service for restarting the ApplicationMaster container on failure. The per-application ApplicationMaster has the responsibility of negotiating appropriate resource containers from the Scheduler, tracking their status and monitoring for progress.
-MRV2 maintains **API compatibility** with previous stable release (hadoop-1.x). This means that all Map-Reduce jobs should still run unchanged on top of MRv2 with just a recompile.
+MapReduce in hadoop-2.x maintains **API compatibility** with previous stable release (hadoop-1.x). This means that all MapReduce jobs should still run unchanged on top of YARN with just a recompile.
[11/21] hadoop git commit: MAPREDUCE-6427. Fix typo in
JobHistoryEventHandler. Contributed by Ray Chiang
Posted by aw...@apache.org.
MAPREDUCE-6427. Fix typo in JobHistoryEventHandler. Contributed by Ray Chiang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4ccdb11
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4ccdb11
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4ccdb11
Branch: refs/heads/HADOOP-12111
Commit: f4ccdb11dca17db139a3746584e321d884651d01
Parents: 979c9ca
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Jul 14 14:51:06 2015 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Jul 14 14:55:43 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 2 ++
.../apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ccdb11/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 95eec1c..31f4eaa 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -537,6 +537,8 @@ Release 2.8.0 - UNRELEASED
RMContainerAllocator.reduceNodeLabelExpression.
(Brahma Reddy Battula via aajisaka)
+ MAPREDUCE-6427. Fix typo in JobHistoryEventHandler. (Ray Chiang via cdouglas)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ccdb11/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 0457cc5..b0bcfcd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -748,7 +748,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("JOB_CONF_PATH", jse.getJobConfPath());
tEvent.addEventInfo("ACLS", jse.getJobAcls());
tEvent.addEventInfo("JOB_QUEUE_NAME", jse.getJobQueueName());
- tEvent.addEventInfo("WORKLFOW_ID", jse.getWorkflowId());
+ tEvent.addEventInfo("WORKFLOW_ID", jse.getWorkflowId());
tEvent.addEventInfo("WORKFLOW_NAME", jse.getWorkflowName());
tEvent.addEventInfo("WORKFLOW_NAME_NAME", jse.getWorkflowNodeName());
tEvent.addEventInfo("WORKFLOW_ADJACENCIES",
[02/21] hadoop git commit: Add HDFS-8143 to CHANGES.txt.
Posted by aw...@apache.org.
Add HDFS-8143 to CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7c8311e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7c8311e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7c8311e
Branch: refs/heads/HADOOP-12111
Commit: f7c8311e9836ad1a1a2ef6eca8b42fd61a688164
Parents: 2466460
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Mon Jul 13 14:59:45 2015 -0700
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Mon Jul 13 14:59:45 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7c8311e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4fa566d..1491990 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1081,6 +1081,9 @@ Release 2.7.1 - 2015-07-06
HDFS-7164. Feature documentation for HDFS-6581. (Arpit Agarwal)
+ HDFS-8143. Mover should exit after some retry when failed to move blocks.
+ (Surendra Singh Lilhore via szetszwo)
+
OPTIMIZATIONS
HDFS-8480. Fix performance and timeout issues in HDFS-7929 by using
hard-links to preserve old edit logs, instead of copying them. (Zhe Zhang
[20/21] hadoop git commit: Merge branch 'trunk' into HADOOP-12111
Posted by aw...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b67f74b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b67f74b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b67f74b4
Branch: refs/heads/HADOOP-12111
Commit: b67f74b41885e10020e14579ee9b9afe8158f742
Parents: 33f2feb 1ba2986
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jul 16 10:33:53 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jul 16 10:33:53 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 16 ++
.../apache/hadoop/fs/ByteBufferReadable.java | 4 +
.../hadoop/io/compress/bzip2/Bzip2Factory.java | 2 +-
.../java/org/apache/hadoop/util/SysInfo.java | 14 ++
.../org/apache/hadoop/util/SysInfoLinux.java | 157 +++++++++++++-
.../org/apache/hadoop/util/SysInfoWindows.java | 12 ++
.../apache/hadoop/util/hash/JenkinsHash.java | 15 +-
...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 5 +-
.../hadoop/io/TestSequenceFileAppend.java | 5 +
.../apache/hadoop/test/GenericTestUtils.java | 9 +
.../apache/hadoop/util/TestSysInfoLinux.java | 77 ++++++-
.../org/apache/hadoop/hdfs/inotify/Event.java | 37 +++-
.../src/main/proto/inotify.proto | 7 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 19 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 1 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 17 ++
.../hadoop/hdfs/server/balancer/Dispatcher.java | 18 ++
.../hdfs/server/datanode/BlockReceiver.java | 34 +--
.../apache/hadoop/hdfs/server/mover/Mover.java | 27 ++-
.../namenode/InotifyFSEditLogOpTranslator.java | 4 +
.../hdfs/TestDFSInotifyEventInputStream.java | 17 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 43 +++-
.../TestBlockReportRateLimiting.java | 64 ++----
.../hadoop/hdfs/server/mover/TestMover.java | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 2 +
.../jobhistory/JobHistoryEventHandler.java | 2 +-
hadoop-project/pom.xml | 2 +-
hadoop-project/src/site/site.xml | 2 +-
.../gridmix/DummyResourceCalculatorPlugin.java | 17 ++
hadoop-yarn-project/CHANGES.txt | 15 +-
.../hadoop/yarn/event/AsyncDispatcher.java | 24 +--
.../yarn/util/ResourceCalculatorPlugin.java | 18 ++
.../hadoop/yarn/event/DrainDispatcher.java | 13 +-
.../hadoop/yarn/event/TestAsyncDispatcher.java | 61 ------
.../scheduler/fair/FSLeafQueue.java | 9 +-
.../scheduler/fair/FairScheduler.java | 40 ++--
.../scheduler/fair/SchedulingPolicy.java | 11 +
.../DominantResourceFairnessPolicy.java | 18 +-
.../fair/policies/FairSharePolicy.java | 11 +-
.../scheduler/fair/policies/FifoPolicy.java | 15 +-
.../scheduler/fair/TestFSLeafQueue.java | 64 ++++++
.../scheduler/fair/TestFairScheduler.java | 207 ++++++++++++++++---
.../src/site/markdown/NodeManager.md | 49 ++++-
.../src/site/markdown/NodeManagerRestart.md | 53 -----
.../hadoop-yarn-site/src/site/markdown/YARN.md | 22 +-
45 files changed, 949 insertions(+), 312 deletions(-)
----------------------------------------------------------------------
[05/21] hadoop git commit: YARN-3453. Ensure preemption logic in
FairScheduler uses DominantResourceCalculator in DRF queues to prevent
unnecessary thrashing. (asuresh)
Posted by aw...@apache.org.
YARN-3453. Ensure preemption logic in FairScheduler uses DominantResourceCalculator in DRF queues to prevent unnecessary thrashing. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac94ba3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac94ba3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac94ba3e
Branch: refs/heads/HADOOP-12111
Commit: ac94ba3e185115b83351e35c610c2b8ff91b1ebc
Parents: a431ed9
Author: Arun Suresh <as...@apache.org>
Authored: Tue Jul 14 00:23:55 2015 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Tue Jul 14 00:23:55 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/fair/FSLeafQueue.java | 9 +-
.../scheduler/fair/FairScheduler.java | 40 ++--
.../scheduler/fair/SchedulingPolicy.java | 11 +
.../DominantResourceFairnessPolicy.java | 18 +-
.../fair/policies/FairSharePolicy.java | 11 +-
.../scheduler/fair/policies/FifoPolicy.java | 15 +-
.../scheduler/fair/TestFSLeafQueue.java | 64 ++++++
.../scheduler/fair/TestFairScheduler.java | 207 ++++++++++++++++---
9 files changed, 317 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5c17f04..780c667 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -628,6 +628,9 @@ Release 2.8.0 - UNRELEASED
YARN-3381. Fix typo InvalidStateTransitonException.
(Brahma Reddy Battula via aajisaka)
+ YARN-3453. Ensure preemption logic in FairScheduler uses DominantResourceCalculator
+ in DRF queues to prevent unnecessary thrashing. (asuresh)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 6779a1b..f90a198 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -560,9 +560,10 @@ public class FSLeafQueue extends FSQueue {
}
private boolean isStarved(Resource share) {
- Resource desiredShare = Resources.min(scheduler.getResourceCalculator(),
- scheduler.getClusterResource(), share, getDemand());
- return Resources.lessThan(scheduler.getResourceCalculator(),
- scheduler.getClusterResource(), getResourceUsage(), desiredShare);
+ Resource desiredShare = Resources.min(policy.getResourceCalculator(),
+ scheduler.getClusterResource(), share, getDemand());
+ Resource resourceUsage = getResourceUsage();
+ return Resources.lessThan(policy.getResourceCalculator(),
+ scheduler.getClusterResource(), resourceUsage, desiredShare);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index cbc10e7..efe6544 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -371,10 +371,9 @@ public class FairScheduler extends
Resource resToPreempt = Resources.clone(Resources.none());
for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
- Resources.addTo(resToPreempt, resToPreempt(sched, curTime));
+ Resources.addTo(resToPreempt, resourceDeficit(sched, curTime));
}
- if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, resToPreempt,
- Resources.none())) {
+ if (isResourceGreaterThanNone(resToPreempt)) {
preemptResources(resToPreempt);
}
}
@@ -404,8 +403,7 @@ public class FairScheduler extends
RMContainer container = warnedIter.next();
if ((container.getState() == RMContainerState.RUNNING ||
container.getState() == RMContainerState.ALLOCATED) &&
- Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
- toPreempt, Resources.none())) {
+ isResourceGreaterThanNone(toPreempt)) {
warnOrKillContainer(container);
Resources.subtractFrom(toPreempt, container.getContainer().getResource());
} else {
@@ -419,8 +417,7 @@ public class FairScheduler extends
queue.resetPreemptedResources();
}
- while (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
- toPreempt, Resources.none())) {
+ while (isResourceGreaterThanNone(toPreempt)) {
RMContainer container =
getQueueManager().getRootQueue().preemptContainer();
if (container == null) {
@@ -442,7 +439,11 @@ public class FairScheduler extends
long duration = getClock().getTime() - start;
fsOpDurations.addPreemptCallDuration(duration);
}
-
+
+ private boolean isResourceGreaterThanNone(Resource toPreempt) {
+ return (toPreempt.getMemory() > 0) || (toPreempt.getVirtualCores() > 0);
+ }
+
protected void warnOrKillContainer(RMContainer container) {
ApplicationAttemptId appAttemptId = container.getApplicationAttemptId();
FSAppAttempt app = getSchedulerApp(appAttemptId);
@@ -485,33 +486,34 @@ public class FairScheduler extends
* max of the two amounts (this shouldn't happen unless someone sets the
* timeouts to be identical for some reason).
*/
- protected Resource resToPreempt(FSLeafQueue sched, long curTime) {
+ protected Resource resourceDeficit(FSLeafQueue sched, long curTime) {
long minShareTimeout = sched.getMinSharePreemptionTimeout();
long fairShareTimeout = sched.getFairSharePreemptionTimeout();
Resource resDueToMinShare = Resources.none();
Resource resDueToFairShare = Resources.none();
+ ResourceCalculator calc = sched.getPolicy().getResourceCalculator();
if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) {
- Resource target = Resources.min(RESOURCE_CALCULATOR, clusterResource,
+ Resource target = Resources.componentwiseMin(
sched.getMinShare(), sched.getDemand());
- resDueToMinShare = Resources.max(RESOURCE_CALCULATOR, clusterResource,
+ resDueToMinShare = Resources.max(calc, clusterResource,
Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
}
if (curTime - sched.getLastTimeAtFairShareThreshold() > fairShareTimeout) {
- Resource target = Resources.min(RESOURCE_CALCULATOR, clusterResource,
- sched.getFairShare(), sched.getDemand());
- resDueToFairShare = Resources.max(RESOURCE_CALCULATOR, clusterResource,
+ Resource target = Resources.componentwiseMin(
+ sched.getFairShare(), sched.getDemand());
+ resDueToFairShare = Resources.max(calc, clusterResource,
Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
}
- Resource resToPreempt = Resources.max(RESOURCE_CALCULATOR, clusterResource,
+ Resource deficit = Resources.max(calc, clusterResource,
resDueToMinShare, resDueToFairShare);
- if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
- resToPreempt, Resources.none())) {
- String message = "Should preempt " + resToPreempt + " res for queue "
+ if (Resources.greaterThan(calc, clusterResource,
+ deficit, Resources.none())) {
+ String message = "Should preempt " + deficit + " res for queue "
+ sched.getName() + ": resDueToMinShare = " + resDueToMinShare
+ ", resDueToFairShare = " + resDueToFairShare;
LOG.info(message);
}
- return resToPreempt;
+ return deficit;
}
public synchronized RMContainerTokenSecretManager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
index abdc834..160ba4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
@@ -26,6 +26,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
+
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+
import java.util.Collection;
import java.util.Comparator;
import java.util.concurrent.ConcurrentHashMap;
@@ -98,6 +101,14 @@ public abstract class SchedulingPolicy {
public void initialize(Resource clusterCapacity) {}
/**
+ * The {@link ResourceCalculator} returned by this method should be used
+ * for any calculations involving resources.
+ *
+ * @return ResourceCalculator instance to use
+ */
+ public abstract ResourceCalculator getResourceCalculator();
+
+ /**
* @return returns the name of {@link SchedulingPolicy}
*/
public abstract String getName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 86d503b..45fbf98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -29,6 +29,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
+
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import static org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType.*;
@@ -44,8 +47,10 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
public static final String NAME = "DRF";
- private DominantResourceFairnessComparator comparator =
+ private static final DominantResourceFairnessComparator COMPARATOR =
new DominantResourceFairnessComparator();
+ private static final DominantResourceCalculator CALCULATOR =
+ new DominantResourceCalculator();
@Override
public String getName() {
@@ -59,9 +64,14 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
@Override
public Comparator<Schedulable> getComparator() {
- return comparator;
+ return COMPARATOR;
}
-
+
+ @Override
+ public ResourceCalculator getResourceCalculator() {
+ return CALCULATOR;
+ }
+
@Override
public void computeShares(Collection<? extends Schedulable> schedulables,
Resource totalResources) {
@@ -105,7 +115,7 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
@Override
public void initialize(Resource clusterCapacity) {
- comparator.setClusterCapacity(clusterCapacity);
+ COMPARATOR.setClusterCapacity(clusterCapacity);
}
public static class DominantResourceFairnessComparator implements Comparator<Schedulable> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 918db9d..3b9f07f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
@@ -43,7 +44,8 @@ public class FairSharePolicy extends SchedulingPolicy {
public static final String NAME = "fair";
private static final DefaultResourceCalculator RESOURCE_CALCULATOR =
new DefaultResourceCalculator();
- private FairShareComparator comparator = new FairShareComparator();
+ private static final FairShareComparator COMPARATOR =
+ new FairShareComparator();
@Override
public String getName() {
@@ -111,7 +113,12 @@ public class FairSharePolicy extends SchedulingPolicy {
@Override
public Comparator<Schedulable> getComparator() {
- return comparator;
+ return COMPARATOR;
+ }
+
+ @Override
+ public ResourceCalculator getResourceCalculator() {
+ return RESOURCE_CALCULATOR;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
index 7d88933..a644e58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
@@ -27,6 +27,10 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
+
+
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
@@ -36,7 +40,9 @@ import com.google.common.annotations.VisibleForTesting;
public class FifoPolicy extends SchedulingPolicy {
@VisibleForTesting
public static final String NAME = "FIFO";
- private FifoComparator comparator = new FifoComparator();
+ private static final FifoComparator COMPARATOR = new FifoComparator();
+ private static final DefaultResourceCalculator CALCULATOR =
+ new DefaultResourceCalculator();
@Override
public String getName() {
@@ -68,7 +74,12 @@ public class FifoPolicy extends SchedulingPolicy {
@Override
public Comparator<Schedulable> getComparator() {
- return comparator;
+ return COMPARATOR;
+ }
+
+ @Override
+ public ResourceCalculator getResourceCalculator() {
+ return CALCULATOR;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 385ea0b..7637410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -233,6 +233,70 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
assertFalse(queueB2.isStarvedForFairShare());
}
+ @Test (timeout = 5000)
+ public void testIsStarvedForFairShareDRF() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<weight>.5</weight>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<weight>.5</weight>");
+ out.println("</queue>");
+ out.println("<defaultFairSharePreemptionThreshold>1</defaultFairSharePreemptionThreshold>");
+ out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
+ out.println("</allocations>");
+ out.close();
+
+ resourceManager = new MockRM(conf);
+ resourceManager.start();
+ scheduler = (FairScheduler) resourceManager.getResourceScheduler();
+
+ // Add one big node (only care about aggregate capacity)
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024, 10), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ scheduler.update();
+
+ // Queue A wants 7 * 1024, 1. Node update gives this all to A
+ createSchedulingRequest(7 * 1024, 1, "queueA", "user1", 1);
+ scheduler.update();
+ NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(nodeEvent2);
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue queueA = queueMgr.getLeafQueue("queueA", false);
+ assertEquals(7 * 1024, queueA.getResourceUsage().getMemory());
+ assertEquals(1, queueA.getResourceUsage().getVirtualCores());
+
+ // Queue B has 3 reqs :
+ // 1) 2 * 1024, 5 .. which will be granted
+ // 2) 1 * 1024, 1 .. which will be granted
+ // 3) 1 * 1024, 1 .. which wont
+ createSchedulingRequest(2 * 1024, 5, "queueB", "user1", 1);
+ createSchedulingRequest(1 * 1024, 2, "queueB", "user1", 2);
+ scheduler.update();
+ for (int i = 0; i < 3; i ++) {
+ scheduler.handle(nodeEvent2);
+ }
+
+ FSLeafQueue queueB = queueMgr.getLeafQueue("queueB", false);
+ assertEquals(3 * 1024, queueB.getResourceUsage().getMemory());
+ assertEquals(6, queueB.getResourceUsage().getVirtualCores());
+
+ scheduler.update();
+
+ // Verify that Queue us not starved for fair share..
+ // Since the Starvation logic now uses DRF when the policy = drf, The
+ // Queue should not be starved
+ assertFalse(queueB.isStarvedForFairShare());
+ }
+
@Test
public void testConcurrentAccess() {
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac94ba3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 56e8adc..2260f73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -100,7 +100,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ControlledClock;
-import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Assert;
@@ -1706,8 +1705,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
clock.tickSec(11);
scheduler.update();
- Resource toPreempt = scheduler.resToPreempt(scheduler.getQueueManager()
- .getLeafQueue("queueA.queueA2", false), clock.getTime());
+ Resource toPreempt = scheduler.resourceDeficit(scheduler.getQueueManager()
+ .getLeafQueue("queueA.queueA2", false), clock.getTime());
assertEquals(3277, toPreempt.getMemory());
// verify if the 3 containers required by queueA2 are preempted in the same
@@ -1829,25 +1828,173 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.getQueueManager().getLeafQueue("queueD", true);
assertTrue(Resources.equals(
- Resources.none(), scheduler.resToPreempt(schedC, clock.getTime())));
+ Resources.none(), scheduler.resourceDeficit(schedC, clock.getTime())));
assertTrue(Resources.equals(
- Resources.none(), scheduler.resToPreempt(schedD, clock.getTime())));
+ Resources.none(), scheduler.resourceDeficit(schedD, clock.getTime())));
// After minSharePreemptionTime has passed, they should want to preempt min
// share.
clock.tickSec(6);
assertEquals(
- 1024, scheduler.resToPreempt(schedC, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(schedC, clock.getTime()).getMemory());
assertEquals(
- 1024, scheduler.resToPreempt(schedD, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(schedD, clock.getTime()).getMemory());
// After fairSharePreemptionTime has passed, they should want to preempt
// fair share.
scheduler.update();
clock.tickSec(6);
assertEquals(
- 1536 , scheduler.resToPreempt(schedC, clock.getTime()).getMemory());
+ 1536 , scheduler.resourceDeficit(schedC, clock.getTime()).getMemory());
assertEquals(
- 1536, scheduler.resToPreempt(schedD, clock.getTime()).getMemory());
+ 1536, scheduler.resourceDeficit(schedD, clock.getTime()).getMemory());
+ }
+
+ @Test
+/**
+ * Tests the timing of decision to preempt tasks.
+ */
+ public void testPreemptionDecisionWithDRF() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ ControlledClock clock = new ControlledClock();
+ scheduler.setClock(clock);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"default\">");
+ out.println("<maxResources>0mb,0vcores</maxResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<weight>.25</weight>");
+ out.println("<minResources>1024mb,1vcores</minResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<weight>.25</weight>");
+ out.println("<minResources>1024mb,2vcores</minResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueC\">");
+ out.println("<weight>.25</weight>");
+ out.println("<minResources>1024mb,3vcores</minResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueD\">");
+ out.println("<weight>.25</weight>");
+ out.println("<minResources>1024mb,2vcores</minResources>");
+ out.println("</queue>");
+ out.println("<defaultMinSharePreemptionTimeout>5</defaultMinSharePreemptionTimeout>");
+ out.println("<defaultFairSharePreemptionTimeout>10</defaultFairSharePreemptionTimeout>");
+ out.println("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>");
+ out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Create four nodes
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 4), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ RMNode node2 =
+ MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 4), 2,
+ "127.0.0.2");
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ RMNode node3 =
+ MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 4), 3,
+ "127.0.0.3");
+ NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3);
+ scheduler.handle(nodeEvent3);
+
+ // Queue A and B each request three containers
+ ApplicationAttemptId app1 =
+ createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 1);
+ ApplicationAttemptId app2 =
+ createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 2);
+ ApplicationAttemptId app3 =
+ createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 3);
+
+ ApplicationAttemptId app4 =
+ createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 1);
+ ApplicationAttemptId app5 =
+ createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 2);
+ ApplicationAttemptId app6 =
+ createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 3);
+
+ scheduler.update();
+
+ // Sufficient node check-ins to fully schedule containers
+ for (int i = 0; i < 2; i++) {
+ NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(nodeUpdate1);
+
+ NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
+ scheduler.handle(nodeUpdate2);
+
+ NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
+ scheduler.handle(nodeUpdate3);
+ }
+
+ // Now new requests arrive from queues C and D
+ ApplicationAttemptId app7 =
+ createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 1);
+ ApplicationAttemptId app8 =
+ createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 2);
+ ApplicationAttemptId app9 =
+ createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 3);
+
+ ApplicationAttemptId app10 =
+ createSchedulingRequest(1 * 1024, "queueD", "user1", 2, 1);
+ ApplicationAttemptId app11 =
+ createSchedulingRequest(1 * 1024, "queueD", "user1", 2, 2);
+ ApplicationAttemptId app12 =
+ createSchedulingRequest(1 * 1024, "queueD", "user1", 2, 3);
+
+ scheduler.update();
+
+ FSLeafQueue schedC =
+ scheduler.getQueueManager().getLeafQueue("queueC", true);
+ FSLeafQueue schedD =
+ scheduler.getQueueManager().getLeafQueue("queueD", true);
+
+ assertTrue(Resources.equals(
+ Resources.none(), scheduler.resourceDeficit(schedC, clock.getTime())));
+ assertTrue(Resources.equals(
+ Resources.none(), scheduler.resourceDeficit(schedD, clock.getTime())));
+
+ // Test :
+ // 1) whether componentWise min works as expected.
+ // 2) DRF calculator is used
+
+ // After minSharePreemptionTime has passed, they should want to preempt min
+ // share.
+ clock.tickSec(6);
+ Resource res = scheduler.resourceDeficit(schedC, clock.getTime());
+ assertEquals(1024, res.getMemory());
+ // Demand = 3
+ assertEquals(3, res.getVirtualCores());
+
+ res = scheduler.resourceDeficit(schedD, clock.getTime());
+ assertEquals(1024, res.getMemory());
+ // Demand = 6, but min share = 2
+ assertEquals(2, res.getVirtualCores());
+
+ // After fairSharePreemptionTime has passed, they should want to preempt
+ // fair share.
+ scheduler.update();
+ clock.tickSec(6);
+ res = scheduler.resourceDeficit(schedC, clock.getTime());
+ assertEquals(1536, res.getMemory());
+ assertEquals(3, res.getVirtualCores());
+
+ res = scheduler.resourceDeficit(schedD, clock.getTime());
+ assertEquals(1536, res.getMemory());
+ // Demand = 6, but fair share = 3
+ assertEquals(3, res.getVirtualCores());
}
@Test
@@ -1964,71 +2111,71 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSLeafQueue queueC = queueMgr.getLeafQueue("queueC", true);
assertTrue(Resources.equals(
- Resources.none(), scheduler.resToPreempt(queueB1, clock.getTime())));
+ Resources.none(), scheduler.resourceDeficit(queueB1, clock.getTime())));
assertTrue(Resources.equals(
- Resources.none(), scheduler.resToPreempt(queueB2, clock.getTime())));
+ Resources.none(), scheduler.resourceDeficit(queueB2, clock.getTime())));
assertTrue(Resources.equals(
- Resources.none(), scheduler.resToPreempt(queueC, clock.getTime())));
+ Resources.none(), scheduler.resourceDeficit(queueC, clock.getTime())));
// After 5 seconds, queueB1 wants to preempt min share
scheduler.update();
clock.tickSec(6);
assertEquals(
- 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemory());
assertEquals(
- 0, scheduler.resToPreempt(queueB2, clock.getTime()).getMemory());
+ 0, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemory());
assertEquals(
- 0, scheduler.resToPreempt(queueC, clock.getTime()).getMemory());
+ 0, scheduler.resourceDeficit(queueC, clock.getTime()).getMemory());
// After 10 seconds, queueB2 wants to preempt min share
scheduler.update();
clock.tickSec(5);
assertEquals(
- 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemory());
assertEquals(
- 1024, scheduler.resToPreempt(queueB2, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemory());
assertEquals(
- 0, scheduler.resToPreempt(queueC, clock.getTime()).getMemory());
+ 0, scheduler.resourceDeficit(queueC, clock.getTime()).getMemory());
// After 15 seconds, queueC wants to preempt min share
scheduler.update();
clock.tickSec(5);
assertEquals(
- 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemory());
assertEquals(
- 1024, scheduler.resToPreempt(queueB2, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemory());
assertEquals(
- 1024, scheduler.resToPreempt(queueC, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueC, clock.getTime()).getMemory());
// After 20 seconds, queueB2 should want to preempt fair share
scheduler.update();
clock.tickSec(5);
assertEquals(
- 1024, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemory());
assertEquals(
- 1536, scheduler.resToPreempt(queueB2, clock.getTime()).getMemory());
+ 1536, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemory());
assertEquals(
- 1024, scheduler.resToPreempt(queueC, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueC, clock.getTime()).getMemory());
// After 25 seconds, queueB1 should want to preempt fair share
scheduler.update();
clock.tickSec(5);
assertEquals(
- 1536, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory());
+ 1536, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemory());
assertEquals(
- 1536, scheduler.resToPreempt(queueB2, clock.getTime()).getMemory());
+ 1536, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemory());
assertEquals(
- 1024, scheduler.resToPreempt(queueC, clock.getTime()).getMemory());
+ 1024, scheduler.resourceDeficit(queueC, clock.getTime()).getMemory());
// After 30 seconds, queueC should want to preempt fair share
scheduler.update();
clock.tickSec(5);
assertEquals(
- 1536, scheduler.resToPreempt(queueB1, clock.getTime()).getMemory());
+ 1536, scheduler.resourceDeficit(queueB1, clock.getTime()).getMemory());
assertEquals(
- 1536, scheduler.resToPreempt(queueB2, clock.getTime()).getMemory());
+ 1536, scheduler.resourceDeficit(queueB2, clock.getTime()).getMemory());
assertEquals(
- 1536, scheduler.resToPreempt(queueC, clock.getTime()).getMemory());
+ 1536, scheduler.resourceDeficit(queueC, clock.getTime()).getMemory());
}
@Test
[21/21] hadoop git commit: HADOOP-12226. CHANGED_MODULES is wrong for
ant (aw)
Posted by aw...@apache.org.
HADOOP-12226. CHANGED_MODULES is wrong for ant (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/840e0e5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/840e0e5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/840e0e5f
Branch: refs/heads/HADOOP-12111
Commit: 840e0e5f7808790e53cbb67bccf7e216b2f84034
Parents: b67f74b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jul 16 10:36:36 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jul 16 10:36:36 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 224 +++++++++++++++++++++++++++--------------
1 file changed, 151 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/840e0e5f/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 327e6f1..24dafc3 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -84,8 +84,12 @@ function setup_defaults
PATCH_BRANCH=""
PATCH_BRANCH_DEFAULT="master"
- #shellcheck disable=SC2034
+ # shellcheck disable=SC2034
CHANGED_MODULES=""
+ # shellcheck disable=SC2034
+ CHANGED_UNFILTERED_MODULES=""
+ # shellcheck disable=SC2034
+ CHANGED_UNION_MODULES=""
USER_MODULE_LIST=""
OFFLINE=false
CHANGED_FILES=""
@@ -656,10 +660,12 @@ function echo_and_redirect
find "${BASEDIR}" -type d -exec chmod +x {} \;
# to the screen
+ echo "cd $(pwd)"
echo "${*} > ${logfile} 2>&1"
# to the log
- echo "${*}" > "${logfile}"
- # the actual command
+ echo "cd $(pwd)" > "${logfile}"
+ echo "${*}" >> "${logfile}"
+ # run the actual command
"${@}" >> "${logfile}" 2>&1
}
@@ -723,6 +729,7 @@ function testpatch_usage
echo "--project=<name> The short name for project currently using test-patch (default 'yetus')"
echo "--resetrepo Forcibly clean the repo"
echo "--run-tests Run all relevant tests below the base directory"
+ echo "--skip-dirs=<list> Skip following directories for module finding"
echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d"
echo "--summarize=<bool> Allow tests to summarize results"
echo "--testlist=<list> Specify which subsystem tests to use (comma delimited)"
@@ -768,6 +775,7 @@ function parse_args
{
local i
local j
+ local testlist
for i in "$@"; do
case ${i} in
@@ -893,6 +901,11 @@ function parse_args
--run-tests)
RUN_TESTS=true
;;
+ --skip-dirs=*)
+ MODULE_SKIPDIRS=${i#*=}
+ MODULE_SKIPDIRS=${MODULE_SKIPDIRS//,/ }
+ yetus_debug "Setting skipdirs to ${MODULE_SKIPDIRS}"
+ ;;
--skip-system-plugins)
LOAD_SYSTEM_PLUGINS=false
;;
@@ -1007,50 +1020,28 @@ function parse_args
GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
}
-## @description Locate the pom.xml file for a given directory
-## @audience private
-## @stability stable
-## @replaceable no
-## @return directory containing the pom.xml. Nothing returned if not found.
-function find_pomxml_dir
-{
- local dir=$1
-
- yetus_debug "Find pom.xml dir for: ${dir}"
-
- while builtin true; do
- if [[ -f "${dir}/pom.xml" ]];then
- echo "${dir}"
- yetus_debug "Found: ${dir}"
- return
- elif [[ ${dir} == "." ]]; then
- yetus_error "ERROR: pom.xml is not found. Make sure the target is a Maven-based project."
- return
- else
- dir=$(dirname "${dir}")
- fi
- done
-}
-
-## @description Locate the build.xml file for a given directory
+## @description Locate the build file for a given directory
## @audience private
## @stability stable
## @replaceable no
-## @return directory containing the build.xml. Nothing returned if not found.
-function find_buildxml_dir
+## @return directory containing the buildfile. Nothing returned if not found.
+## @params buildfile
+## @params directory
+function find_buildfile_dir
{
- local dir=$1
+ local buildfile=$1
+ local dir=$2
- yetus_debug "Find build.xml dir for: ${dir}"
+ yetus_debug "Find ${buildfile} dir for: ${dir}"
while builtin true; do
- if [[ -f "${dir}/build.xml" ]];then
+ if [[ -f "${dir}/${buildfile}" ]];then
echo "${dir}"
yetus_debug "Found: ${dir}"
- return
+ return 0
elif [[ ${dir} == "." ]]; then
- yetus_error "ERROR: build.xml is not found. Make sure the target is a Ant-based project."
- return
+ yetus_debug "ERROR: ${buildfile} is not found."
+ return 1
else
dir=$(dirname "${dir}")
fi
@@ -1073,6 +1064,42 @@ function find_changed_files
/^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
}
+## @description Check for directories to skip during
+## @description changed module calcuation
+## @audience private
+## @stability stable
+## @replaceable no
+## @params directory
+## @returns 0 for use
+## @returns 1 for skip
+function module_skipdir
+{
+ local dir=${1}
+ local i
+
+ yetus_debug "Checking skipdirs for ${dir}"
+
+ if [[ -z ${MODULE_SKIPDIRS} ]]; then
+ yetus_debug "Skipping skipdirs"
+ return 0
+ fi
+
+ while builtin true; do
+ for i in ${MODULE_SKIPDIRS}; do
+ if [[ ${dir} = "${i}" ]];then
+ yetus_debug "Found a skip: ${dir}"
+ return 1
+ fi
+ done
+ if [[ ${dir} == "." ]]; then
+ return 0
+ else
+ dir=$(dirname "${dir}")
+ yetus_debug "Trying to skip: ${dir}"
+ fi
+ done
+}
+
## @description Find the modules of the build that ${PATCH_DIR}/patch modifies
## @audience private
## @stability stable
@@ -1082,60 +1109,111 @@ function find_changed_modules
{
local i
local changed_dirs
- local pomdirs
- local pomdir
+ local builddirs
+ local builddir
local module
- local pommods
+ local buildmods
+ local prev_builddir
+ local i=1
+ local dir
+ local buildfile
+
+ case ${BUILDTOOL} in
+ maven)
+ buildfile=pom.xml
+ ;;
+ ant)
+ buildfile=build.xml
+ ;;
+ *)
+ yetus_error "ERROR: Unsupported build tool."
+ output_to_console 1
+ output_to_bugsystem 1
+ cleanup_and_exit 1
+ ;;
+ esac
changed_dirs=$(for i in ${CHANGED_FILES}; do dirname "${i}"; done | sort -u)
# Now find all the modules that were changed
for i in ${changed_dirs}; do
- case ${BUILDTOOL} in
- maven)
- #shellcheck disable=SC2086
- pomdir=$(find_pomxml_dir ${i})
- if [[ -z ${pomdir} ]]; then
- output_to_console 1
- output_to_bugsystem 1
- cleanup_and_exit 1
- fi
- pomdirs="${pomdirs} ${pomdir}"
- ;;
- ant)
- #shellcheck disable=SC2086
- pomdir=$(find_buildxml_dir ${i})
- if [[ -z ${pomdir} ]]; then
- output_to_console 1
- output_to_bugsystem 1
- cleanup_and_exit 1
- fi
- pomdirs="${pomdirs} ${pomdir}"
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- output_to_console 1
- output_to_bugsystem 1
- cleanup_and_exit 1
- ;;
- esac
+
+ module_skipdir "${i}"
+ if [[ $? != 0 ]]; then
+ continue
+ fi
+
+ builddir=$(find_buildfile_dir ${buildfile} "${i}")
+ if [[ -z ${builddir} ]]; then
+ yetus_error "ERROR: ${buildfile} is not found. Make sure the target is a ${BUILDTOOL}-based project."
+ output_to_console 1
+ output_to_bugsystem 1
+ cleanup_and_exit 1
+ fi
+ builddirs="${builddirs} ${builddir}"
done
#shellcheck disable=SC2086,SC2034
- CHANGED_UNFILTERED_MODULES=$(echo ${pomdirs} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
+ CHANGED_UNFILTERED_MODULES=$(echo ${builddirs} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
+ #shellcheck disable=SC2086,SC2116
+ CHANGED_UNFILTERED_MODULES=$(echo ${CHANGED_UNFILTERED_MODULES})
+
- if [[ ${BUILDTOOL} == maven ]]; then
+ if [[ ${BUILDTOOL} = maven
+ && ${QETESTMODE} = false ]]; then
# Filter out modules without code
- for module in ${pomdirs}; do
+ for module in ${builddirs}; do
${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null
if [[ "$?" != 0 ]]; then
- pommods="${pommods} ${module}"
+ buildmods="${buildmods} ${module}"
fi
done
+ elif [[ ${QETESTMODE} = true ]]; then
+ buildmods=${builddirs}
fi
#shellcheck disable=SC2086,SC2034
- CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
+ CHANGED_MODULES=$(echo ${buildmods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
+
+ # turn it back into a list so that anyone printing doesn't
+ # generate multiline output
+ #shellcheck disable=SC2086,SC2116
+ CHANGED_MODULES=$(echo ${CHANGED_MODULES})
+
+ yetus_debug "Locate the union of ${CHANGED_MODULES}"
+ # shellcheck disable=SC2086
+ count=$(echo ${CHANGED_MODULES} | wc -w)
+ if [[ ${count} -lt 2 ]]; then
+ yetus_debug "Only one entry, so keeping it ${CHANGED_MODULES}"
+ # shellcheck disable=SC2034
+ CHANGED_UNION_MODULES=${CHANGED_MODULES}
+ return
+ fi
+
+ i=1
+ while [[ ${i} -lt 100 ]]
+ do
+ module=$(echo "${CHANGED_MODULES}" | tr ' ' '\n' | cut -f1-${i} -d/ | uniq)
+ count=$(echo "${module}" | wc -w)
+ if [[ ${count} -eq 1
+ && -f ${module}/${buildfile} ]]; then
+ prev_builddir=${module}
+ elif [[ ${count} -gt 1 ]]; then
+ builddir=${prev_builddir}
+ break
+ fi
+ ((i=i+1))
+ done
+
+ if [[ -z ${builddir} ]]; then
+ builddir="."
+ fi
+
+ yetus_debug "Finding union of ${builddir}"
+ builddir=$(find_buildfile_dir ${buildfile} "${builddir}" || true)
+
+ #shellcheck disable=SC2034
+ CHANGED_UNION_MODULES="${builddir}"
}
## @description git checkout the appropriate branch to test. Additionally, this calls
[16/21] hadoop git commit: HADOOP-10615. FileInputStream in
JenkinsHash#main() is never closed. Contributed by Chen He.
Posted by aw...@apache.org.
HADOOP-10615. FileInputStream in JenkinsHash#main() is never closed. Contributed by Chen He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/111e6a3f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/111e6a3f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/111e6a3f
Branch: refs/heads/HADOOP-12111
Commit: 111e6a3fdf613767782817836c42810bf2bda5e8
Parents: 3ec0a04
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Thu Jul 16 14:08:31 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Thu Jul 16 14:08:31 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/util/hash/JenkinsHash.java | 15 ++++++++-------
2 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/111e6a3f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3d64156..9bb7a26 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -961,6 +961,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12200. TestCryptoStreamsWithOpensslAesCtrCryptoCodec should be
skipped in non-native profile. (Masatake Iwasaki via aajisaka)
+ HADOOP-10615. FileInputStream in JenkinsHash#main() is never closed.
+ (Chen He via ozawa)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/111e6a3f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
index bf48913..f3895d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
@@ -252,13 +252,14 @@ public class JenkinsHash extends Hash {
System.err.println("Usage: JenkinsHash filename");
System.exit(-1);
}
- FileInputStream in = new FileInputStream(args[0]);
- byte[] bytes = new byte[512];
- int value = 0;
- JenkinsHash hash = new JenkinsHash();
- for (int length = in.read(bytes); length > 0 ; length = in.read(bytes)) {
- value = hash.hash(bytes, length, value);
+ try (FileInputStream in = new FileInputStream(args[0])) {
+ byte[] bytes = new byte[512];
+ int value = 0;
+ JenkinsHash hash = new JenkinsHash();
+ for (int length = in.read(bytes); length > 0; length = in.read(bytes)) {
+ value = hash.hash(bytes, length, value);
+ }
+ System.out.println(Math.abs(value));
}
- System.out.println(Math.abs(value));
}
}
[12/21] hadoop git commit: HADOOP-12232. Upgrade Tomcat dependency to
6.0.44. Contributed by Chris Nauroth.
Posted by aw...@apache.org.
HADOOP-12232. Upgrade Tomcat dependency to 6.0.44. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a16ee60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a16ee60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a16ee60
Branch: refs/heads/HADOOP-12111
Commit: 0a16ee60174b15e3df653bb107cb2d0c2d606330
Parents: f4ccdb1
Author: cnauroth <cn...@apache.org>
Authored: Tue Jul 14 14:53:08 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Tue Jul 14 15:22:33 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
hadoop-project/pom.xml | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a16ee60/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a9bd7de..a807d12 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -966,6 +966,8 @@ Release 2.7.2 - UNRELEASED
IMPROVEMENTS
+ HADOOP-12232. Upgrade Tomcat dependency to 6.0.44. (cnauroth)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a16ee60/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index e010de1..d563420 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -76,7 +76,7 @@
<curator.version>2.7.1</curator.version>
<findbugs.version>3.0.0</findbugs.version>
- <tomcat.version>6.0.41</tomcat.version>
+ <tomcat.version>6.0.44</tomcat.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.7</javac.version>
[18/21] hadoop git commit: HADOOP-12240. Fix tests requiring native
library to be skipped in non-native profile. Contributed by Masatake Iwasaki.
Posted by aw...@apache.org.
HADOOP-12240. Fix tests requiring native library to be skipped in non-native profile. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90bda9c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90bda9c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90bda9c6
Branch: refs/heads/HADOOP-12111
Commit: 90bda9c6110f0344104b8ae96ea6999770ebb391
Parents: f02dd14
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Thu Jul 16 16:04:55 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Thu Jul 16 16:04:55 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java | 5 ++---
.../java/org/apache/hadoop/io/TestSequenceFileAppend.java | 5 +++++
.../test/java/org/apache/hadoop/test/GenericTestUtils.java | 9 +++++++++
4 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90bda9c6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9bb7a26..cf79bab 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -964,6 +964,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10615. FileInputStream in JenkinsHash#main() is never closed.
(Chen He via ozawa)
+ HADOOP-12240. Fix tests requiring native library to be skipped in non-native
+ profile. (Masatake Iwasaki via ozawa)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90bda9c6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index d5f25b8..9bbdf0a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -19,19 +19,18 @@ package org.apache.hadoop.crypto;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.BeforeClass;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assume.assumeTrue;
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
extends TestCryptoStreams {
@BeforeClass
public static void init() throws Exception {
- assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ GenericTestUtils.assumeNativeCodeLoaded();
Configuration conf = new Configuration();
conf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90bda9c6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
index 4576642..7aa305f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.SequenceFile.Writer.Option;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.serializer.JavaSerializationComparator;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -140,6 +141,7 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendRecordCompression() throws Exception {
+ GenericTestUtils.assumeNativeCodeLoaded();
Path file = new Path(ROOT_PATH, "testseqappendblockcompr.seq");
fs.delete(file, true);
@@ -173,6 +175,7 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendBlockCompression() throws Exception {
+ GenericTestUtils.assumeNativeCodeLoaded();
Path file = new Path(ROOT_PATH, "testseqappendblockcompr.seq");
fs.delete(file, true);
@@ -247,6 +250,8 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendSort() throws Exception {
+ GenericTestUtils.assumeNativeCodeLoaded();
+
Path file = new Path(ROOT_PATH, "testseqappendSort.seq");
fs.delete(file, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90bda9c6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 65c18d1..7d52a8a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -35,6 +35,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Layout;
@@ -43,6 +44,7 @@ import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.WriterAppender;
import org.junit.Assert;
+import org.junit.Assume;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -410,4 +412,11 @@ public abstract class GenericTestUtils {
}
}
}
+
+ /**
+ * Skip test if native code is not loaded.
+ */
+ public static void assumeNativeCodeLoaded() {
+ Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
+ }
}
[17/21] hadoop git commit: YARN-3174. Consolidate the NodeManager and
NodeManagerRestart documentation into one. Contributed by Masatake Iwasaki.
Posted by aw...@apache.org.
YARN-3174. Consolidate the NodeManager and NodeManagerRestart documentation into one. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f02dd146
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f02dd146
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f02dd146
Branch: refs/heads/HADOOP-12111
Commit: f02dd146f58bcfa0595eec7f2433bafdd857630f
Parents: 111e6a3
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Thu Jul 16 15:22:30 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Thu Jul 16 15:22:30 2015 +0900
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 ++
.../src/site/markdown/NodeManager.md | 41 +++++++++++++--
.../src/site/markdown/NodeManagerRestart.md | 53 --------------------
4 files changed, 42 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02dd146/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 55be0d9..ee0dfcd 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -124,7 +124,7 @@
<item name="Web Application Proxy" href="hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html"/>
<item name="Timeline Server" href="hadoop-yarn/hadoop-yarn-site/TimelineServer.html"/>
<item name="Writing YARN Applications" href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>
- <item name="NodeManager Restart" href="hadoop-yarn/hadoop-yarn-site/NodeManagerRestart.html"/>
+ <item name="NodeManager" href="hadoop-yarn/hadoop-yarn-site/NodeManager.html"/>
<item name="DockerContainerExecutor" href="hadoop-yarn/hadoop-yarn-site/DockerContainerExecutor.html"/>
<item name="Using CGroups" href="hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html"/>
<item name="Secure Containers" href="hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02dd146/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0a6f871..1e6c7d5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -631,6 +631,9 @@ Release 2.8.0 - UNRELEASED
YARN-3453. Ensure preemption logic in FairScheduler uses DominantResourceCalculator
in DRF queues to prevent unnecessary thrashing. (asuresh)
+ YARN-3174. Consolidate the NodeManager and NodeManagerRestart documentation
+ into one. (Masatake Iwasaki via ozawa)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02dd146/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
index 6341c60..69e99a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
@@ -12,19 +12,23 @@
limitations under the License. See accompanying LICENSE file.
-->
-NodeManager Overview
-=====================
+NodeManager
+===========
* [Overview](#Overview)
* [Health Checker Service](#Health_checker_service)
* [Disk Checker](#Disk_Checker)
* [External Health Script](#External_Health_Script)
+* [NodeManager Restart](#NodeManager_Restart)
+ * [Introduction](#Introduction)
+ * [Enabling NM Restart](#Enabling_NM_Restart)
Overview
--------
The NodeManager is responsible for launching and managing containers on a node. Containers execute tasks as specified by the AppMaster.
+
Health Checker Service
----------------------
@@ -42,7 +46,6 @@ The NodeManager runs services to determine the health of the node it is executin
| `yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage` | Float between 0-100 | The maximum percentage of disk space that may be utilized before a disk is marked as unhealthy by the disk checker service. This check is run for every disk used by the NodeManager. The default value is 100 i.e. the entire disk can be used. |
| `yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb` | Integer | The minimum amount of free space that must be available on the disk for the disk checker service to mark the disk as healthy. This check is run for every disk used by the NodeManager. The default value is 0 i.e. the entire disk can be used. |
-
###External Health Script
Users may specify their own health checker script that will be invoked by the health checker service. Users may specify a timeout as well as options to be passed to the script. If the script exits with a non-zero exit code, times out or results in an exception being thrown, the node is marked as unhealthy. Please note that if the script cannot be executed due to permissions or an incorrect path, etc, then it counts as a failure and the node will be reported as unhealthy. Please note that speifying a health check script is not mandatory. If no script is specified, only the disk checker status will be used to determine the health of the node. The following configuration parameters can be used to set the health script:
@@ -55,3 +58,35 @@ The NodeManager runs services to determine the health of the node it is executin
| `yarn.nodemanager.health-checker.script.opts` | String | Arguments to be passed to the script when the script is executed. |
+NodeManager Restart
+-------------------
+
+### Introduction
+
+This document gives an overview of NodeManager (NM) restart, a feature that enables the NodeManager to be restarted without losing the active containers running on the node. At a high level, the NM stores any necessary state to a local state-store as it processes container-management requests. When the NM restarts, it recovers by first loading state for various subsystems and then letting those subsystems perform recovery using the loaded state.
+
+### Enabling NM Restart
+
+Step 1. To enable NM Restart functionality, set the following property in **conf/yarn-site.xml** to *true*.
+
+| Property | Value |
+|:---- |:---- |
+| `yarn.nodemanager.recovery.enabled` | `true`, (default value is set to false) |
+
+Step 2. Configure a path to the local file-system directory where the NodeManager can save its run state.
+
+| Property | Description |
+|:---- |:---- |
+| `yarn.nodemanager.recovery.dir` | The local filesystem directory in which the node manager will store state when recovery is enabled. The default value is set to `$hadoop.tmp.dir/yarn-nm-recovery`. |
+
+Step 3. Configure a valid RPC address for the NodeManager.
+
+| Property | Description |
+|:---- |:---- |
+| `yarn.nodemanager.address` | Ephemeral ports (port 0, which is default) cannot be used for the NodeManager's RPC server specified via yarn.nodemanager.address as it can make NM use different ports before and after a restart. This will break any previously running clients that were communicating with the NM before restart. Explicitly setting yarn.nodemanager.address to an address with specific port number (for e.g 0.0.0.0:45454) is a precondition for enabling NM restart. |
+
+Step 4. Auxiliary services.
+
+ * NodeManagers in a YARN cluster can be configured to run auxiliary services. For a completely functional NM restart, YARN relies on any auxiliary service configured to also support recovery. This usually includes (1) avoiding usage of ephemeral ports so that previously running clients (in this case, usually containers) are not disrupted after restart and (2) having the auxiliary service itself support recoverability by reloading any previous state when NodeManager restarts and reinitializes the auxiliary service.
+
+ * A simple example for the above is the auxiliary service 'ShuffleHandler' for MapReduce (MR). ShuffleHandler respects the above two requirements already, so users/admins don't have do anything for it to support NM restart: (1) The configuration property **mapreduce.shuffle.port** controls which port the ShuffleHandler on a NodeManager host binds to, and it defaults to a non-ephemeral port. (2) The ShuffleHandler service also already supports recovery of previous state after NM restarts.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f02dd146/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRestart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRestart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRestart.md
deleted file mode 100644
index be7d75b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManagerRestart.md
+++ /dev/null
@@ -1,53 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-NodeManager Restart
-===================
-
-* [Introduction](#Introduction)
-* [Enabling NM Restart](#Enabling_NM_Restart)
-
-Introduction
-------------
-
-This document gives an overview of NodeManager (NM) restart, a feature that enables the NodeManager to be restarted without losing the active containers running on the node. At a high level, the NM stores any necessary state to a local state-store as it processes container-management requests. When the NM restarts, it recovers by first loading state for various subsystems and then letting those subsystems perform recovery using the loaded state.
-
-Enabling NM Restart
--------------------
-
-Step 1. To enable NM Restart functionality, set the following property in **conf/yarn-site.xml** to *true*.
-
-| Property | Value |
-|:---- |:---- |
-| `yarn.nodemanager.recovery.enabled` | `true`, (default value is set to false) |
-
-Step 2. Configure a path to the local file-system directory where the NodeManager can save its run state.
-
-| Property | Description |
-|:---- |:---- |
-| `yarn.nodemanager.recovery.dir` | The local filesystem directory in which the node manager will store state when recovery is enabled. The default value is set to `$hadoop.tmp.dir/yarn-nm-recovery`. |
-
-Step 3. Configure a valid RPC address for the NodeManager.
-
-| Property | Description |
-|:---- |:---- |
-| `yarn.nodemanager.address` | Ephemeral ports (port 0, which is default) cannot be used for the NodeManager's RPC server specified via yarn.nodemanager.address as it can make NM use different ports before and after a restart. This will break any previously running clients that were communicating with the NM before restart. Explicitly setting yarn.nodemanager.address to an address with specific port number (for e.g 0.0.0.0:45454) is a precondition for enabling NM restart. |
-
-Step 4. Auxiliary services.
-
- * NodeManagers in a YARN cluster can be configured to run auxiliary services. For a completely functional NM restart, YARN relies on any auxiliary service configured to also support recovery. This usually includes (1) avoiding usage of ephemeral ports so that previously running clients (in this case, usually containers) are not disrupted after restart and (2) having the auxiliary service itself support recoverability by reloading any previous state when NodeManager restarts and reinitializes the auxiliary service.
-
- * A simple example for the above is the auxiliary service 'ShuffleHandler' for MapReduce (MR). ShuffleHandler respects the above two requirements already, so users/admins don't have do anything for it to support NM restart: (1) The configuration property **mapreduce.shuffle.port** controls which port the ShuffleHandler on a NodeManager host binds to, and it defaults to a non-ephemeral port. (2) The ShuffleHandler service also already supports recovery of previous state after NM restarts.
-
-
[03/21] hadoop git commit: HDFS-8541. Mover should exit with
NO_MOVE_PROGRESS if there is no move progress. Contributed by Surendra Singh
Lilhore
Posted by aw...@apache.org.
HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move progress. Contributed by Surendra Singh Lilhore
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ef03a4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ef03a4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ef03a4c
Branch: refs/heads/HADOOP-12111
Commit: 9ef03a4c5bb5573eadc7d04e371c4af2dc6bae37
Parents: f7c8311
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Mon Jul 13 15:12:26 2015 -0700
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Mon Jul 13 15:12:26 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/balancer/Dispatcher.java | 18 +++++++++++++
.../apache/hadoop/hdfs/server/mover/Mover.java | 27 +++++++++++++++-----
.../hadoop/hdfs/server/mover/TestMover.java | 2 +-
4 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef03a4c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1491990..e843dcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -716,6 +716,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8751. Remove setBlocks API from INodeFile and misc code cleanup. (Zhe
Zhang via jing9)
+ HDFS-8541. Mover should exit with NO_MOVE_PROGRESS if there is no move
+ progress. (Surendra Singh Lilhore via szetszwo)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef03a4c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index 4a8f40f..298b86d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -317,6 +317,7 @@ public class Dispatcher {
sendRequest(out, eb, accessToken);
receiveResponse(in);
nnc.getBytesMoved().addAndGet(block.getNumBytes());
+ target.getDDatanode().setHasSuccess();
LOG.info("Successfully moved " + this);
} catch (IOException e) {
LOG.warn("Failed to move " + this + ": " + e.getMessage());
@@ -500,6 +501,7 @@ public class Dispatcher {
/** blocks being moved but not confirmed yet */
private final List<PendingMove> pendings;
private volatile boolean hasFailure = false;
+ private volatile boolean hasSuccess = false;
private final int maxConcurrentMoves;
@Override
@@ -573,6 +575,10 @@ public class Dispatcher {
void setHasFailure() {
this.hasFailure = true;
}
+
+ void setHasSuccess() {
+ this.hasSuccess = true;
+ }
}
/** A node that can be the sources of a block move */
@@ -965,6 +971,18 @@ public class Dispatcher {
}
/**
+ * @return true if some moves are success.
+ */
+ public static boolean checkForSuccess(
+ Iterable<? extends StorageGroup> targets) {
+ boolean hasSuccess = false;
+ for (StorageGroup t : targets) {
+ hasSuccess |= t.getDDatanode().hasSuccess;
+ }
+ return hasSuccess;
+ }
+
+ /**
* Decide if the block is a good candidate to be moved from source to target.
* A block is a good candidate if
* 1. the block is not in the process of being moved/has not been moved;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef03a4c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 344b9fc..afacebb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -269,10 +269,14 @@ public class Mover {
// wait for pending move to finish and retry the failed migration
boolean hasFailed = Dispatcher.waitForMoveCompletion(storages.targets
.values());
- if (hasFailed) {
+ boolean hasSuccess = Dispatcher.checkForSuccess(storages.targets
+ .values());
+ if (hasFailed && !hasSuccess) {
if (retryCount.get() == retryMaxAttempts) {
- throw new IOException("Failed to move some block's after "
+ result.setRetryFailed();
+ LOG.error("Failed to move some block's after "
+ retryMaxAttempts + " retries.");
+ return result;
} else {
retryCount.incrementAndGet();
}
@@ -713,10 +717,12 @@ public class Mover {
private boolean hasRemaining;
private boolean noBlockMoved;
+ private boolean retryFailed;
Result() {
hasRemaining = false;
noBlockMoved = true;
+ retryFailed = false;
}
boolean isHasRemaining() {
@@ -735,16 +741,25 @@ public class Mover {
this.noBlockMoved = noBlockMoved;
}
+ void setRetryFailed() {
+ this.retryFailed = true;
+ }
+
/**
- * @return SUCCESS if all moves are success and there is no remaining move.
+ * @return NO_MOVE_PROGRESS if no progress in move after some retry. Return
+ * SUCCESS if all moves are success and there is no remaining move.
* Return NO_MOVE_BLOCK if there moves available but all the moves
* cannot be scheduled. Otherwise, return IN_PROGRESS since there
* must be some remaining moves.
*/
ExitStatus getExitStatus() {
- return !isHasRemaining() ? ExitStatus.SUCCESS
- : isNoBlockMoved() ? ExitStatus.NO_MOVE_BLOCK
- : ExitStatus.IN_PROGRESS;
+ if (retryFailed) {
+ return ExitStatus.NO_MOVE_PROGRESS;
+ } else {
+ return !isHasRemaining() ? ExitStatus.SUCCESS
+ : isNoBlockMoved() ? ExitStatus.NO_MOVE_BLOCK
+ : ExitStatus.IN_PROGRESS;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef03a4c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index 899b5c0..d3d814c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -404,7 +404,7 @@ public class TestMover {
int rc = ToolRunner.run(conf, new Mover.Cli(),
new String[] {"-p", file.toString()});
Assert.assertEquals("Movement should fail after some retry",
- ExitStatus.IO_EXCEPTION.getExitCode(), rc);
+ ExitStatus.NO_MOVE_PROGRESS.getExitCode(), rc);
} finally {
cluster.shutdown();
}