You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/03 22:25:08 UTC
[1/5] hadoop git commit: HDFS-8245. Standby namenode doesn't process
DELETED_BLOCK if the addblock request is in edit log. Contributed by Rushabh
S Shah. (cherry picked from commit 2d4ae3d18bc530fa9f81ee616db8af3395705fb9)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6.1 7b1a71a7a -> 563dbd29e
HDFS-8245. Standby namenode doesn't process DELETED_BLOCK if the addblock request is in edit log. Contributed by Rushabh S Shah.
(cherry picked from commit 2d4ae3d18bc530fa9f81ee616db8af3395705fb9)
(cherry picked from commit f264a5aeede7e144af11f5357c7f901993de8e12)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
(cherry picked from commit 470019e9b88e0fcede926442b91d102b595c7ace)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3ca4dfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3ca4dfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3ca4dfa
Branch: refs/heads/branch-2.6.1
Commit: b3ca4dfaff627a04a1a07aa305e1d30651e66056
Parents: 7b1a71a
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri May 8 16:37:26 2015 -0500
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 12:19:21 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 24 ++++-
.../server/datanode/TestBlockReplacement.java | 97 ++++++++++++++++++++
.../hdfs/server/namenode/ha/TestDNFencing.java | 4 -
4 files changed, 121 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ca4dfa/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5ed6cbf..f59f673 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -138,6 +138,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7894. Rolling upgrade readiness is not updated in jmx until query
command is issued. (Brahma Reddy Battula via kihwal)
+ HDFS-8254. Standby namenode doesn't process DELETED_BLOCK if the add block
+ request is in edit log. (Rushabh S Shah via kihwal)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ca4dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index e5d97d1..b69c78e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2287,8 +2287,15 @@ public class BlockManager {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing previouly queued message " + rbi);
}
- processAndHandleReportedBlock(rbi.getStorageInfo(),
- rbi.getBlock(), rbi.getReportedState(), null);
+ if (rbi.getReportedState() == null) {
+ // This is a DELETE_BLOCK request
+ DatanodeStorageInfo storageInfo = rbi.getStorageInfo();
+ removeStoredBlock(rbi.getBlock(),
+ storageInfo.getDatanodeDescriptor());
+ } else {
+ processAndHandleReportedBlock(rbi.getStorageInfo(),
+ rbi.getBlock(), rbi.getReportedState(), null);
+ }
}
}
@@ -2984,6 +2991,17 @@ public class BlockManager {
}
}
+ private void removeStoredBlock(DatanodeStorageInfo storageInfo, Block block,
+ DatanodeDescriptor node) {
+ if (shouldPostponeBlocksFromFuture &&
+ namesystem.isGenStampInFuture(block)) {
+ queueReportedBlock(storageInfo, block, null,
+ QUEUE_REASON_FUTURE_GENSTAMP);
+ return;
+ }
+ removeStoredBlock(block, node);
+ }
+
/**
* Modify (block-->datanode) map. Possibly generate replication tasks, if the
* removed block is still valid.
@@ -3171,7 +3189,7 @@ public class BlockManager {
for (ReceivedDeletedBlockInfo rdbi : srdb.getBlocks()) {
switch (rdbi.getStatus()) {
case DELETED_BLOCK:
- removeStoredBlock(rdbi.getBlock(), node);
+ removeStoredBlock(storageInfo, rdbi.getBlock(), node);
deleted++;
break;
case RECEIVED_BLOCK:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ca4dfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
index e0d7964..86b77d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
@@ -42,7 +42,9 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.client.BlockReportOptions;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -51,8 +53,11 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Time;
import org.junit.Test;
@@ -280,6 +285,98 @@ public class TestBlockReplacement {
}
/**
+ * Standby namenode doesn't queue Delete block request when the add block
+ * request is in the edit log which are yet to be read.
+ * @throws Exception
+ */
+ @Test
+ public void testDeletedBlockWhenAddBlockIsInEdit() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(1).build();
+ DFSClient client = null;
+ try {
+ cluster.waitActive();
+ assertEquals("Number of namenodes is not 2", 2,
+ cluster.getNumNameNodes());
+ // Transitioning the namenode 0 to active.
+ cluster.transitionToActive(0);
+ assertTrue("Namenode 0 should be in active state",
+ cluster.getNameNode(0).isActiveState());
+ assertTrue("Namenode 1 should be in standby state",
+ cluster.getNameNode(1).isStandbyState());
+
+ // Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover
+ // to true.
+ DataNodeTestUtils.triggerHeartbeat(cluster.getDataNodes().get(0));
+ FileSystem fs = cluster.getFileSystem(0);
+
+ // Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale
+ // to false.
+ cluster.getDataNodes().get(0).triggerBlockReport(
+ new BlockReportOptions.Factory().setIncremental(false).build());
+
+ Path fileName = new Path("/tmp.txt");
+ // create a file with one block
+ DFSTestUtil.createFile(fs, fileName, 10L, (short)1, 1234L);
+ DFSTestUtil.waitReplication(fs,fileName, (short)1);
+
+ client = new DFSClient(cluster.getFileSystem(0).getUri(), conf);
+ List<LocatedBlock> locatedBlocks = client.getNamenode().
+ getBlockLocations("/tmp.txt", 0, 10L).getLocatedBlocks();
+ assertTrue(locatedBlocks.size() == 1);
+ assertTrue(locatedBlocks.get(0).getLocations().length == 1);
+
+ // add a second datanode to the cluster
+ cluster.startDataNodes(conf, 1, true, null, null, null, null);
+ assertEquals("Number of datanodes should be 2", 2,
+ cluster.getDataNodes().size());
+
+ DataNode dn0 = cluster.getDataNodes().get(0);
+ DataNode dn1 = cluster.getDataNodes().get(1);
+ String activeNNBPId = cluster.getNamesystem(0).getBlockPoolId();
+ DatanodeDescriptor sourceDnDesc = NameNodeAdapter.getDatanode(
+ cluster.getNamesystem(0), dn0.getDNRegistrationForBP(activeNNBPId));
+ DatanodeDescriptor destDnDesc = NameNodeAdapter.getDatanode(
+ cluster.getNamesystem(0), dn1.getDNRegistrationForBP(activeNNBPId));
+
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
+
+ LOG.info("replaceBlock: " + replaceBlock(block,
+ (DatanodeInfo)sourceDnDesc, (DatanodeInfo)sourceDnDesc,
+ (DatanodeInfo)destDnDesc));
+ // Waiting for the FsDatasetAsyncDsikService to delete the block
+ Thread.sleep(3000);
+ // Triggering the incremental block report to report the deleted block to
+ // namnemode
+ cluster.getDataNodes().get(0).triggerBlockReport(
+ new BlockReportOptions.Factory().setIncremental(true).build());
+
+ cluster.transitionToStandby(0);
+ cluster.transitionToActive(1);
+
+ assertTrue("Namenode 1 should be in active state",
+ cluster.getNameNode(1).isActiveState());
+ assertTrue("Namenode 0 should be in standby state",
+ cluster.getNameNode(0).isStandbyState());
+ client.close();
+
+ // Opening a new client for new active namenode
+ client = new DFSClient(cluster.getFileSystem(1).getUri(), conf);
+ List<LocatedBlock> locatedBlocks1 = client.getNamenode()
+ .getBlockLocations("/tmp.txt", 0, 10L).getLocatedBlocks();
+
+ assertEquals(1, locatedBlocks1.size());
+ assertEquals("The block should be only on 1 datanode ", 1,
+ locatedBlocks1.get(0).getLocations().length);
+ } finally {
+ IOUtils.cleanup(null, client);
+ cluster.shutdown();
+ }
+ }
+
+ /**
* @param args
*/
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ca4dfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index a538b6e..3748328 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -156,10 +156,6 @@ public class TestDNFencing {
banner("NN2 Metadata immediately after failover");
doMetasave(nn2);
- // Even though NN2 considers the blocks over-replicated, it should
- // post-pone the block invalidation because the DNs are still "stale".
- assertEquals(30, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
-
banner("Triggering heartbeats and block reports so that fencing is completed");
cluster.triggerHeartbeats();
cluster.triggerBlockReports();
[5/5] hadoop git commit: HDFS-8431. hdfs crypto class not found in
Windows. Contributed by Anu Engineer.
Posted by vi...@apache.org.
HDFS-8431. hdfs crypto class not found in Windows. Contributed by Anu Engineer.
(cherry picked from commit 50eeea13000f0c82e0567410f0f8b611248f8c1b)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
(cherry picked from commit 25db34127811fbadb9a698fa3a76e24d426fb0f6)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
(cherry picked from commit 77a10e76e99c14cd26ebb3664304f6ed9cc7bf65)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/563dbd29
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/563dbd29
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/563dbd29
Branch: refs/heads/branch-2.6.1
Commit: 563dbd29eb91f198448117b446fb5963d9606e44
Parents: 42ce052
Author: cnauroth <cn...@apache.org>
Authored: Wed May 27 22:54:00 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 12:59:35 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd | 7 ++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/563dbd29/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 715bef4..7d88967 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -144,6 +144,9 @@ Release 2.6.1 - UNRELEASED
HDFS-8404. Pending block replication can get stuck using older genstamp
(Nathan Roberts via kihwal)
+ HDFS-8431. hdfs crypto class not found in Windows.
+ (Anu Engineer via cnauroth)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/563dbd29/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index 69424ed..453a023 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -47,7 +47,7 @@ if "%1" == "--config" (
goto print_usage
)
- set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies
+ set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies crypto
for %%i in ( %hdfscommands% ) do (
if %hdfs-command% == %%i set hdfscommand=true
)
@@ -159,6 +159,10 @@ goto :eof
set CLASS=org.apache.hadoop.hdfs.tools.GetStoragePolicies
goto :eof
+:crypto
+ set CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
+ goto :eof
+
@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
:make_command_arguments
if "%1" == "--config" (
@@ -207,6 +211,7 @@ goto :eof
@echo lsSnapshottableDir list all snapshottable dirs owned by the current user
@echo Use -help to see options
@echo cacheadmin configure the HDFS cache
+ @echo crypto configure HDFS encryption zones
@echo mover run a utility to move block replicas across storage types
@echo storagepolicies get all the existing block storage policies
@echo.
[4/5] hadoop git commit: YARN-3700. Made generic history service load
a number of latest applications according to the parameter or the
configuration. Contributed by Xuan Gong.
Posted by vi...@apache.org.
YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.
(cherry picked from commit 54504133f41e36eaea6bb06c7b9ddb249468ecd7)
(cherry picked from commit 839f81a6326b2f8b3d5183178382c1551b0bc259)
(cherry picked from commit 058380d9ef35f35e8c624fb8783eac0904c4d1f5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42ce0525
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42ce0525
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42ce0525
Branch: refs/heads/branch-2.6.1
Commit: 42ce052585dcb3ff5e954c41409c2a75d507bbb2
Parents: 0f33fcd
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed May 27 16:51:48 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 12:59:33 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 9 ++++++
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../src/main/resources/yarn-default.xml | 9 ++++++
.../ApplicationHistoryClientService.java | 2 +-
.../ApplicationHistoryManager.java | 12 +++++---
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 13 +++++---
.../TestApplicationHistoryClientService.java | 26 ++++++++++++++--
...pplicationHistoryManagerOnTimelineStore.java | 12 ++++----
.../hadoop/yarn/server/webapp/AppsBlock.java | 6 ++++
.../hadoop/yarn/server/webapp/WebServices.java | 31 ++++++++------------
.../src/site/apt/TimelineServer.apt.vm | 8 +++++
13 files changed, 97 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e2419d6..328a0ed 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -144,6 +144,9 @@ Release 2.6.1 - UNRELEASED
YARN-2766. Made ApplicationHistoryManager return a sorted list of apps,
attempts and containers. (Robert Kanter via zjshen)
+ YARN-3700. Made generic history service load a number of latest applications
+ according to the parameter or the configuration. (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6d9c0da..bc89ed0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1285,6 +1285,15 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
+ /**
+ * Defines the max number of applications could be fetched using
+ * REST API or application history protocol and shown in timeline
+ * server web ui.
+ */
+ public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ APPLICATION_HISTORY_PREFIX + "max-applications";
+ public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
TIMELINE_SERVICE_PREFIX + "store-class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 055f895..74b2415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -33,6 +33,7 @@ public interface YarnWebParams {
String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner";
String APP_STATE = "app.state";
+ String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String WEB_UI_TYPE = "web.ui.type";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9d37196..e168600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1269,6 +1269,15 @@
</property>
<property>
+ <description>
+ Defines the max number of applications could be fetched using REST API or
+ application history protocol and shown in timeline server web ui.
+ </description>
+ <name>yarn.timeline-service.generic-application-history.max-applications</name>
+ <value>10000</value>
+ </property>
+
+ <property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index 848224d..632853e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -178,7 +178,7 @@ public class ApplicationHistoryClientService extends AbstractService implements
IOException {
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
- history.getAllApplications().values()));
+ history.getApplications(request.getLimit()).values()));
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
index 041c31b..c218eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
@@ -51,16 +51,20 @@ public interface ApplicationHistoryManager {
IOException;
/**
- * This method returns all Application {@link ApplicationReport}s
- *
+ * This method returns the given number of Application
+ * {@link ApplicationReport}s.
+ *
+ * @param appsNum
+ *
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
- Map<ApplicationId, ApplicationReport> getAllApplications()
- throws YarnException, IOException;
+ Map<ApplicationId, ApplicationReport>
+ getApplications(long appsNum) throws YarnException,
+ IOException;
/**
* Application can have multiple application attempts
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index 803dc01..779b182 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws IOException {
Map<ApplicationId, ApplicationHistoryData> histData =
historyStore.getAllApplications();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index f9ea14f..56ad469 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -73,6 +74,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager;
private String serverHttpAddress;
+ private long maxLoadedApplications;
public ApplicationHistoryManagerOnTimelineStore(
TimelineDataManager timelineDataManager,
@@ -86,6 +88,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
protected void serviceInit(Configuration conf) throws Exception {
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ maxLoadedApplications =
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
super.serviceInit(conf);
}
@@ -96,12 +101,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities(
- ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
- null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
- UserGroupInformation.getLoginUser());
+ ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
+ null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
+ EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index b3fb3de..3a6214f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -53,6 +53,7 @@ import org.junit.Test;
public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
+ private static TimelineDataManager dataManager;
@BeforeClass
public static void setup() throws Exception {
@@ -60,7 +61,7 @@ public class TestApplicationHistoryClientService {
TimelineStore store =
TestApplicationHistoryManagerOnTimelineStore.createStore(2);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
- TimelineDataManager dataManager =
+ dataManager =
new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -97,8 +98,27 @@ public class TestApplicationHistoryClientService {
clientService.getApplications(request);
List<ApplicationReport> appReport = response.getApplicationList();
Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.get(0).getApplicationId());
- Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+
+ // Create a historyManager, and set the max_apps can be loaded
+ // as 1.
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ ApplicationHistoryManagerOnTimelineStore historyManager2 =
+ new ApplicationHistoryManagerOnTimelineStore(dataManager,
+ new ApplicationACLsManager(conf));
+ historyManager2.init(conf);
+ historyManager2.start();
+ @SuppressWarnings("resource")
+ ApplicationHistoryClientService clientService2 =
+ new ApplicationHistoryClientService(historyManager2);
+ response = clientService2.getApplications(request);
+ appReport = response.getApplicationList();
+ Assert.assertNotNull(appReport);
+ Assert.assertTrue(appReport.size() == 1);
+ // Expected to get the appReport for application with appId1
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index ec96e0c..c44f22e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -175,8 +175,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue());
- Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
- Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L
+ + app.getApplicationId().getId(), app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 3L
+ + +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
@@ -324,7 +326,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps =
- historyManager.getAllApplications().values();
+ historyManager.getApplications(Long.MAX_VALUE).values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
}
@@ -450,12 +452,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
entity.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
"test diagnostics info");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 161486d..583a0ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@@ -80,6 +81,11 @@ public class AppsBlock extends HtmlBlock {
try {
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance(reqAppStates);
+ String appsNumStr = $(APPS_NUM);
+ if (appsNumStr != null && !appsNumStr.isEmpty()) {
+ long appsNum = Long.parseLong(appsNumStr);
+ request.setLimit(appsNum);
+ }
if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index a02b80f..0d6f166 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -31,6 +31,13 @@ import javax.ws.rs.WebApplicationException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -39,13 +46,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -71,13 +71,11 @@ public class WebServices {
String startedEnd, String finishBegin, String finishEnd,
Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req);
- long num = 0;
- boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
- long countNum = 0;
+ long countNum = Long.MAX_VALUE;
// set values suitable in case both of begin/end not specified
long sBegin = 0;
@@ -86,7 +84,6 @@ public class WebServices {
long fEnd = Long.MAX_VALUE;
if (count != null && !count.isEmpty()) {
- checkCount = true;
countNum = Long.parseLong(count);
if (countNum <= 0) {
throw new BadRequestException("limit value must be greater then 0");
@@ -147,19 +144,20 @@ public class WebServices {
AppsInfo allApps = new AppsInfo();
Collection<ApplicationReport> appReports = null;
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance();
+ request.setLimit(countNum);
try {
if (callerUGI == null) {
// TODO: the request should take the params like what RMWebServices does
// in YARN-1819.
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
appReports = callerUGI.doAs(
new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
@Override
public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(
- GetApplicationsRequest.newInstance()).getApplicationList();
+ return appBaseProt.getApplications(request).getApplicationList();
}
});
}
@@ -168,10 +166,6 @@ public class WebServices {
}
for (ApplicationReport appReport : appReports) {
- if (checkCount && num == countNum) {
- break;
- }
-
if (checkAppStates
&& !appStates.contains(appReport.getYarnApplicationState().toString()
.toLowerCase())) {
@@ -211,7 +205,6 @@ public class WebServices {
AppInfo app = new AppInfo(appReport);
allApps.add(app);
- num++;
}
return allApps;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
index 92c7377..7332b86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
@@ -104,6 +104,14 @@ YARN Timeline Server
</property>
<property>
+ <description>The max number of applications could be fetched by using REST API
+ or application history protocol and shown in timeline server web ui. Defaults
+ to `10000`.</description>
+ <name>yarn.timeline-service.generic-application-history.max-applications</name>
+ <value>10000</value>
+</property>
+
+<property>
<description>Enables cross-origin support (CORS) for web services where
cross-origin web response headers are needed. For example, javascript making
a web services request to the timeline server.</description>
[3/5] hadoop git commit: YARN-2766. Made ApplicationHistoryManager
return a sorted list of apps,
attempts and containers. Contributed by Robert Kanter.
Posted by vi...@apache.org.
YARN-2766. Made ApplicationHistoryManager return a sorted list of apps, attempts and containers. Contributed by Robert Kanter.
(cherry picked from commit 3648cb57c9f018a3a339c26f5a0ca2779485521a)
(cherry picked from commit 53d6c91df925c58e311ec7cd6a3adb952acc02f4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f33fcd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f33fcd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f33fcd5
Branch: refs/heads/branch-2.6.1
Commit: 0f33fcd50772a1af315b104c39b7ebb1d3a15d4e
Parents: fb096b9
Author: Zhijie Shen <zj...@apache.org>
Authored: Thu Nov 13 13:47:42 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 12:58:42 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../ApplicationHistoryManagerOnTimelineStore.java | 7 ++++---
.../TestApplicationHistoryClientService.java | 4 ++--
3 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f33fcd5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 45522c2..e2419d6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -141,6 +141,9 @@ Release 2.6.1 - UNRELEASED
YARN-3526. ApplicationMaster tracking URL is incorrectly redirected
on a QJM cluster. (Weiwei Yang via xgong)
+ YARN-2766. Made ApplicationHistoryManager return a sorted list of apps,
+ attempts and containers. (Robert Kanter via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f33fcd5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index ca7521f..f9ea14f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -102,7 +103,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
- new HashMap<ApplicationId, ApplicationReport>();
+ new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
for (TimelineEntity entity : entities.getEntities()) {
ApplicationReportExt app =
@@ -128,7 +129,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
Long.MAX_VALUE, EnumSet.allOf(Field.class),
UserGroupInformation.getLoginUser());
Map<ApplicationAttemptId, ApplicationAttemptReport> appAttempts =
- new HashMap<ApplicationAttemptId, ApplicationAttemptReport>();
+ new LinkedHashMap<ApplicationAttemptId, ApplicationAttemptReport>();
for (TimelineEntity entity : entities.getEntities()) {
ApplicationAttemptReport appAttempt =
convertToApplicationAttemptReport(entity);
@@ -198,7 +199,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
UserGroupInformation.getLoginUser());
Map<ContainerId, ContainerReport> containers =
- new HashMap<ContainerId, ContainerReport>();
+ new LinkedHashMap<ContainerId, ContainerReport>();
if (entities != null && entities.getEntities() != null) {
for (TimelineEntity entity : entities.getEntities()) {
ContainerReport container = convertToContainerReport(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f33fcd5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index f0c4433..b3fb3de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -168,7 +168,7 @@ public class TestApplicationHistoryClientService {
clientService.getContainers(request);
List<ContainerReport> containers = response.getContainerList();
Assert.assertNotNull(containers);
- Assert.assertEquals(containerId, containers.get(1).getContainerId());
- Assert.assertEquals(containerId1, containers.get(0).getContainerId());
+ Assert.assertEquals(containerId, containers.get(0).getContainerId());
+ Assert.assertEquals(containerId1, containers.get(1).getContainerId());
}
}
[2/5] hadoop git commit: HDFS-8404. Pending block replication can get
stuck using older genstamp. Contributed by Nathan Roberts. (cherry picked
from commit 8860e352c394372e4eb3ebdf82ea899567f34e4e)
Posted by vi...@apache.org.
HDFS-8404. Pending block replication can get stuck using older genstamp. Contributed by Nathan Roberts.
(cherry picked from commit 8860e352c394372e4eb3ebdf82ea899567f34e4e)
(cherry picked from commit 536b9ee6d6e5b8430fda23cbdcfd859c299fa8ad)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
(cherry picked from commit 2d5e60fa12a62463cd54f1b6b0fcb2ccdbd82c42)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb096b97
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb096b97
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb096b97
Branch: refs/heads/branch-2.6.1
Commit: fb096b97cc80791ae6a9962e8acd4d6b108a9be5
Parents: b3ca4df
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 19 13:06:48 2015 -0500
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 12:32:49 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 17 ++--
.../blockmanagement/TestPendingReplication.java | 98 +++++++++++++++++++-
3 files changed, 108 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb096b97/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f59f673..715bef4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -141,6 +141,9 @@ Release 2.6.1 - UNRELEASED
HDFS-8254. Standby namenode doesn't process DELETED_BLOCK if the add block
request is in edit log. (Rushabh S Shah via kihwal)
+ HDFS-8404. Pending block replication can get stuck using older genstamp
+ (Nathan Roberts via kihwal)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb096b97/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index b69c78e..bd9d2f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1695,13 +1695,18 @@ public class BlockManager {
namesystem.writeLock();
try {
for (int i = 0; i < timedOutItems.length; i++) {
+ /*
+ * Use the blockinfo from the blocksmap to be certain we're working
+ * with the most up-to-date block information (e.g. genstamp).
+ */
+ BlockInfo bi = blocksMap.getStoredBlock(timedOutItems[i]);
+ if (bi == null) {
+ continue;
+ }
NumberReplicas num = countNodes(timedOutItems[i]);
- if (isNeededReplication(timedOutItems[i], getReplication(timedOutItems[i]),
- num.liveReplicas())) {
- neededReplications.add(timedOutItems[i],
- num.liveReplicas(),
- num.decommissionedReplicas(),
- getReplication(timedOutItems[i]));
+ if (isNeededReplication(bi, getReplication(bi), num.liveReplicas())) {
+ neededReplications.add(bi, num.liveReplicas(),
+ num.decommissionedReplicas(), getReplication(bi));
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb096b97/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
index c63badc..085d5de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.junit.Test;
+import org.mockito.Mockito;
/**
* This class tests the internals of PendingReplicationBlocks.java,
@@ -52,13 +53,11 @@ public class TestPendingReplication {
private static final int DFS_REPLICATION_INTERVAL = 1;
// Number of datanodes in the cluster
private static final int DATANODE_COUNT = 5;
-
@Test
public void testPendingReplication() {
PendingReplicationBlocks pendingReplications;
pendingReplications = new PendingReplicationBlocks(TIMEOUT * 1000);
pendingReplications.start();
-
//
// Add 10 blocks to pendingReplications.
//
@@ -140,8 +139,7 @@ public class TestPendingReplication {
//
// Verify that everything has timed out.
//
- assertEquals("Size of pendingReplications ",
- 0, pendingReplications.size());
+ assertEquals("Size of pendingReplications ", 0, pendingReplications.size());
Block[] timedOut = pendingReplications.getTimedOutBlocks();
assertTrue(timedOut != null && timedOut.length == 15);
for (int i = 0; i < timedOut.length; i++) {
@@ -149,6 +147,98 @@ public class TestPendingReplication {
}
pendingReplications.stop();
}
+
+/* Test that processPendingReplications will use the most recent
+ * blockinfo from the blocksmap by placing a larger genstamp into
+ * the blocksmap.
+ */
+ @Test
+ public void testProcessPendingReplications() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ conf.setLong(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, TIMEOUT);
+ MiniDFSCluster cluster = null;
+ Block block;
+ BlockInfo blockInfo;
+ try {
+ cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build();
+ cluster.waitActive();
+
+ FSNamesystem fsn = cluster.getNamesystem();
+ BlockManager blkManager = fsn.getBlockManager();
+
+ PendingReplicationBlocks pendingReplications =
+ blkManager.pendingReplications;
+ UnderReplicatedBlocks neededReplications = blkManager.neededReplications;
+ BlocksMap blocksMap = blkManager.blocksMap;
+
+ //
+ // Add 1 block to pendingReplications with GenerationStamp = 0.
+ //
+
+ block = new Block(1, 1, 0);
+ blockInfo = new BlockInfo(block, (short) 3);
+
+ pendingReplications.increment(block,
+ DatanodeStorageInfo.toDatanodeDescriptors(
+ DFSTestUtil.createDatanodeStorageInfos(1)));
+ BlockCollection bc = Mockito.mock(BlockCollection.class);
+ Mockito.doReturn((short) 3).when(bc).getBlockReplication();
+ // Place into blocksmap with GenerationStamp = 1
+ blockInfo.setGenerationStamp(1);
+ blocksMap.addBlockCollection(blockInfo, bc);
+
+ assertEquals("Size of pendingReplications ", 1,
+ pendingReplications.size());
+
+ // Add a second block to pendingReplications that has no
+ // corresponding entry in blocksmap
+ block = new Block(2, 2, 0);
+ pendingReplications.increment(block,
+ DatanodeStorageInfo.toDatanodeDescriptors(
+ DFSTestUtil.createDatanodeStorageInfos(1)));
+
+ // verify 2 blocks in pendingReplications
+ assertEquals("Size of pendingReplications ", 2,
+ pendingReplications.size());
+
+ //
+ // Wait for everything to timeout.
+ //
+ while (pendingReplications.size() > 0) {
+ try {
+ Thread.sleep(100);
+ } catch (Exception e) {
+ }
+ }
+
+ //
+ // Verify that block moves to neededReplications
+ //
+ while (neededReplications.size() == 0) {
+ try {
+ Thread.sleep(100);
+ } catch (Exception e) {
+ }
+ }
+
+ // Verify that the generation stamp we will try to replicate
+ // is now 1
+ for (Block b: neededReplications) {
+ assertEquals("Generation stamp is 1 ", 1,
+ b.getGenerationStamp());
+ }
+
+ // Verify size of neededReplications is exactly 1.
+ assertEquals("size of neededReplications is 1 ", 1,
+ neededReplications.size());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
/**
* Test if DatanodeProtocol#blockReceivedAndDeleted can correctly update the