You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/10 22:07:37 UTC
[01/13] hadoop git commit: HDFS-8046. Allow better control of
getContentSummary. Contributed by Kihwal Lee. (cherry picked from commit
285b31e75e51ec8e3a796c2cb0208739368ca9b8)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 d1186dbaf -> ee830b0d2
HDFS-8046. Allow better control of getContentSummary. Contributed by Kihwal Lee.
(cherry picked from commit 285b31e75e51ec8e3a796c2cb0208739368ca9b8)
(cherry picked from commit 7e622076d41a85fc9a8600fb270564a085f5cd83)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfb27d7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfb27d7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfb27d7f
Branch: refs/heads/branch-2.7
Commit: cfb27d7f780b51025f99848c5c1ad1e28806ff9a
Parents: d1186db
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Apr 8 15:39:25 2015 -0500
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 10:53:19 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +++-
.../server/namenode/ContentSummaryComputationContext.java | 10 +++++++---
.../hdfs/server/namenode/FSDirStatAndListingOp.java | 2 +-
.../apache/hadoop/hdfs/server/namenode/FSDirectory.java | 8 ++++++++
5 files changed, 23 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e1785d2..8758cba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -77,6 +77,10 @@ Release 2.7.1 - 2015-07-06
HDFS-7546. Document, and set an accepting default for
dfs.namenode.kerberos.principal.pattern (Harsh J via aw)
+ HDFS-8046. Allow better control of getContentSummary (kihwal)
+
+ OPTIMIZATIONS
+
HDFS-7164. Feature documentation for HDFS-6581. (Arpit Agarwal)
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index a8d7159..3bb8a0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -283,7 +283,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;
public static final String DFS_CONTENT_SUMMARY_LIMIT_KEY = "dfs.content-summary.limit";
- public static final int DFS_CONTENT_SUMMARY_LIMIT_DEFAULT = 0;
+ public static final int DFS_CONTENT_SUMMARY_LIMIT_DEFAULT = 5000;
+ public static final String DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_KEY = "dfs.content-summary.sleep-microsec";
+ public static final long DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT = 500;
public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated";
public static final int DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT = 0;
public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index 31f34b9..5739835 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -32,6 +32,8 @@ public class ContentSummaryComputationContext {
private long nextCountLimit = 0;
private long limitPerRun = 0;
private long yieldCount = 0;
+ private long sleepMilliSec = 0;
+ private int sleepNanoSec = 0;
/**
* Constructor
@@ -43,17 +45,19 @@ public class ContentSummaryComputationContext {
* no limit (i.e. no yielding)
*/
public ContentSummaryComputationContext(FSDirectory dir,
- FSNamesystem fsn, long limitPerRun) {
+ FSNamesystem fsn, long limitPerRun, long sleepMicroSec) {
this.dir = dir;
this.fsn = fsn;
this.limitPerRun = limitPerRun;
this.nextCountLimit = limitPerRun;
this.counts = new ContentCounts.Builder().build();
+ this.sleepMilliSec = sleepMicroSec/1000;
+ this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000);
}
/** Constructor for blocking computation. */
public ContentSummaryComputationContext(BlockStoragePolicySuite bsps) {
- this(null, null, 0);
+ this(null, null, 0, 1000);
this.bsps = bsps;
}
@@ -105,7 +109,7 @@ public class ContentSummaryComputationContext {
fsn.readUnlock();
try {
- Thread.sleep(1);
+ Thread.sleep(sleepMilliSec, sleepNanoSec);
} catch (InterruptedException ie) {
} finally {
// reacquire
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 43c2de3..850b3bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -496,7 +496,7 @@ class FSDirStatAndListingOp {
// processed. 0 means disabled. I.e. blocking for the entire duration.
ContentSummaryComputationContext cscc =
new ContentSummaryComputationContext(fsd, fsd.getFSNamesystem(),
- fsd.getContentCountLimit());
+ fsd.getContentCountLimit(), fsd.getContentSleepMicroSec());
ContentSummary cs = targetNode.computeAndConvertContentSummary(cscc);
fsd.addYieldCount(cscc.getYieldCount());
return cs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb27d7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b887eb3..f74c42a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -136,6 +136,7 @@ public class FSDirectory implements Closeable {
private final int maxDirItems;
private final int lsLimit; // max list limit
private final int contentCountLimit; // max content summary counts per run
+ private final long contentSleepMicroSec;
private final INodeMap inodeMap; // Synchronized by dirLock
private long yieldCount = 0; // keep track of lock yield count.
@@ -264,6 +265,9 @@ public class FSDirectory implements Closeable {
this.contentCountLimit = conf.getInt(
DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY,
DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_DEFAULT);
+ this.contentSleepMicroSec = conf.getLong(
+ DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_KEY,
+ DFSConfigKeys.DFS_CONTENT_SUMMARY_SLEEP_MICROSEC_DEFAULT);
// filesystem limits
this.maxComponentLength = conf.getInt(
@@ -345,6 +349,10 @@ public class FSDirectory implements Closeable {
return contentCountLimit;
}
+ long getContentSleepMicroSec() {
+ return contentSleepMicroSec;
+ }
+
int getInodeXAttrsLimit() {
return inodeXAttrsLimit;
}
[11/13] hadoop git commit: HDFS-7314. When the DFSClient lease cannot
be renewed,
abort open-for-write files rather than the entire DFSClient. (mingma)
Posted by vi...@apache.org.
HDFS-7314. When the DFSClient lease cannot be renewed, abort open-for-write files rather than the entire DFSClient. (mingma)
(cherry picked from commit fbd88f1062f3c4b208724d208e3f501eb196dfab)
(cherry picked from commit 516bbf1c20547dc513126df0d9f0934bb65c10c7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/decf8a66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/decf8a66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/decf8a66
Branch: refs/heads/branch-2.7
Commit: decf8a6653e294fb01531df8094f6b99260fd985
Parents: 6ab1a9c
Author: Ming Ma <mi...@apache.org>
Authored: Thu Jul 16 12:33:57 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:29 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 16 +----
.../org/apache/hadoop/hdfs/LeaseRenewer.java | 12 +++-
.../hadoop/hdfs/TestDFSClientRetries.java | 66 +++++++++++++++++++-
4 files changed, 79 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/decf8a66/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d22f592..d90ec70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -13,6 +13,9 @@ Release 2.7.2 - UNRELEASED
HADOOP-5323. Trash documentation should describe its directory structure and
configurations. (Weiwei Yang via ozawa)
+ HDFS-7314. When the DFSClient lease cannot be renewed, abort open-for-write
+ files rather than the entire DFSClient. (mingma)
+
OPTIMIZATIONS
HDFS-8722. Optimize datanode writes for small writes and flushes (kihwal)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/decf8a66/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 87e34cd..e663873 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -916,23 +916,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
void closeConnectionToNamenode() {
RPC.stopProxy(namenode);
}
-
- /** Abort and release resources held. Ignore all errors. */
- void abort() {
- clientRunning = false;
- closeAllFilesBeingWritten(true);
- try {
- // remove reference to this client and stop the renewer,
- // if there is no more clients under the renewer.
- getLeaseRenewer().closeClient(this);
- } catch (IOException ioe) {
- LOG.info("Exception occurred while aborting the client " + ioe);
- }
- closeConnectionToNamenode();
- }
/** Close/abort all files being written. */
- private void closeAllFilesBeingWritten(final boolean abort) {
+ public void closeAllFilesBeingWritten(final boolean abort) {
for(;;) {
final long inodeId;
final DFSOutputStream out;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/decf8a66/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
index e767501..b60f7e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/LeaseRenewer.java
@@ -211,6 +211,12 @@ class LeaseRenewer {
return renewal;
}
+ /** Used for testing only. */
+ @VisibleForTesting
+ public synchronized void setRenewalTime(final long renewal) {
+ this.renewal = renewal;
+ }
+
/** Add a client. */
private synchronized void addClient(final DFSClient dfsc) {
for(DFSClient c : dfsclients) {
@@ -450,8 +456,12 @@ class LeaseRenewer {
+ (elapsed/1000) + " seconds. Aborting ...", ie);
synchronized (this) {
while (!dfsclients.isEmpty()) {
- dfsclients.get(0).abort();
+ DFSClient dfsClient = dfsclients.get(0);
+ dfsClient.closeAllFilesBeingWritten(true);
+ closeClient(dfsClient);
}
+ //Expire the current LeaseRenewer thread.
+ emptyTime = 0;
}
break;
} catch (IOException ie) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/decf8a66/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 382ad48..0a39cb5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -32,6 +32,7 @@ import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.when;
import java.io.FileNotFoundException;
@@ -63,6 +64,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.LeaseRenewer;
import org.apache.hadoop.hdfs.client.HdfsUtils;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
@@ -355,7 +357,59 @@ public class TestDFSClientRetries {
cluster.shutdown();
}
}
-
+
+ /**
+ * Test DFSClient can continue to function after renewLease RPC
+ * receives SocketTimeoutException.
+ */
+ @Test
+ public void testLeaseRenewSocketTimeout() throws Exception
+ {
+ String file1 = "/testFile1";
+ String file2 = "/testFile2";
+ // Set short retry timeouts so this test runs faster
+ conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10);
+ conf.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, 2 * 1000);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ try {
+ cluster.waitActive();
+ NamenodeProtocols spyNN = spy(cluster.getNameNodeRpc());
+ Mockito.doThrow(new SocketTimeoutException()).when(spyNN).renewLease(
+ Mockito.anyString());
+ DFSClient client = new DFSClient(null, spyNN, conf, null);
+ // Get hold of the lease renewer instance used by the client
+ LeaseRenewer leaseRenewer = client.getLeaseRenewer();
+ leaseRenewer.setRenewalTime(100);
+ OutputStream out1 = client.create(file1, false);
+
+ Mockito.verify(spyNN, timeout(10000).times(1)).renewLease(
+ Mockito.anyString());
+ verifyEmptyLease(leaseRenewer);
+ try {
+ out1.write(new byte[256]);
+ fail("existing output stream should be aborted");
+ } catch (IOException e) {
+ }
+
+ // Verify DFSClient can do read operation after renewLease aborted.
+ client.exists(file2);
+ // Verify DFSClient can do write operation after renewLease no longer
+ // throws SocketTimeoutException.
+ Mockito.doNothing().when(spyNN).renewLease(
+ Mockito.anyString());
+ leaseRenewer = client.getLeaseRenewer();
+ leaseRenewer.setRenewalTime(100);
+ OutputStream out2 = client.create(file2, false);
+ Mockito.verify(spyNN, timeout(10000).times(2)).renewLease(
+ Mockito.anyString());
+ out2.write(new byte[256]);
+ out2.close();
+ verifyEmptyLease(leaseRenewer);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* Test that getAdditionalBlock() and close() are idempotent. This allows
* a client to safely retry a call and still produce a correct
@@ -670,7 +724,15 @@ public class TestDFSClientRetries {
}
return ret;
}
-
+
+ private void verifyEmptyLease(LeaseRenewer leaseRenewer) throws Exception {
+ int sleepCount = 0;
+ while (!leaseRenewer.isEmpty() && sleepCount++ < 20) {
+ Thread.sleep(500);
+ }
+ assertTrue("Lease should be empty.", leaseRenewer.isEmpty());
+ }
+
class DFSClientReader implements Runnable {
DFSClient client;
[04/13] hadoop git commit: HADOOP-11491. HarFs incorrectly declared
as requiring an authority. (Brahma Reddy Battula via gera)
Posted by vi...@apache.org.
HADOOP-11491. HarFs incorrectly declared as requiring an authority. (Brahma Reddy Battula via gera)
(cherry picked from commit f343f8657e2b01773a32c2c7d960dc368954b42e)
(cherry picked from commit 58970d69de8a1662e4548cd6d4ca460dd70562f8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afa8189a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afa8189a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afa8189a
Branch: refs/heads/branch-2.7
Commit: afa8189a07a87d807e8c5aebd4d0260b7b450a02
Parents: d3f41f6
Author: Gera Shegalov <ge...@apache.org>
Authored: Fri May 1 15:44:36 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 10:57:52 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/HarFs.java | 2 +-
.../java/org/apache/hadoop/fs/TestHarFileSystemBasics.java | 7 +++++++
3 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa8189a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9aac0d4..bbd4ef3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -10,6 +10,9 @@ Release 2.7.2 - UNRELEASED
HADOOP-12232. Upgrade Tomcat dependency to 6.0.44. (cnauroth)
+ HADOOP-11491. HarFs incorrectly declared as requiring an authority.
+ (Brahma Reddy Battula via gera)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa8189a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFs.java
index a2369e3..4f5fde8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFs.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration;
public class HarFs extends DelegateToFileSystem {
HarFs(final URI theUri, final Configuration conf)
throws IOException, URISyntaxException {
- super(theUri, new HarFileSystem(), conf, "har", true);
+ super(theUri, new HarFileSystem(), conf, "har", false);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa8189a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
index 577abfd..53507b9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
@@ -398,4 +398,11 @@ public class TestHarFileSystemBasics {
}
}
+ @Test
+ public void testHarFsWithoutAuthority() throws Exception {
+ final URI uri = harFileSystem.getUri();
+ Assert.assertNull("har uri authority not null: " + uri, uri.getAuthority());
+ FileContext.getFileContext(uri, conf);
+ }
+
}
[12/13] hadoop git commit: HADOOP-12280. Skip unit tests based on
maven profile rather than NativeCodeLoader.isNativeCodeLoaded (Masatake
Iwasaki via Colin P. McCabe)
Posted by vi...@apache.org.
HADOOP-12280. Skip unit tests based on maven profile rather than NativeCodeLoader.isNativeCodeLoaded (Masatake Iwasaki via Colin P. McCabe)
(cherry picked from commit 6f83274afc1eba1159427684d72d8f13778c5a88)
(cherry picked from commit e92107b18f82b3501deaa6170d322a0fb512ec71)
(cherry picked from commit 3bd9b7459bfe2e4d81d60498832dc297cd01e003)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb06549e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb06549e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb06549e
Branch: refs/heads/branch-2.7
Commit: fb06549e9da2b65201234ec117d81495e76c6581
Parents: decf8a6
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Aug 4 13:51:04 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:30 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/crypto/TestCryptoCodec.java | 11 +++--------
.../TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java | 2 ++
.../org/apache/hadoop/io/TestSequenceFileAppend.java | 5 +++++
.../java/org/apache/hadoop/test/GenericTestUtils.java | 11 +++++++++++
5 files changed, 24 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06549e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 144c296..49da893 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -352,6 +352,9 @@ Release 2.7.0 - 2015-04-20
HADOOP-11801. Update BUILDING.txt for Ubuntu. (Gabor Liptak via
Arpit Agarwal)
+ HADOOP-12280. Skip unit tests based on maven profile rather than
+ NativeCodeLoader.isNativeCodeLoaded (Masatake Iwasaki via Colin P. McCabe)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06549e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 6e2ceaf..52e547b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.RandomDatum;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Assert;
@@ -69,10 +70,7 @@ public class TestCryptoCodec {
@Test(timeout=120000)
public void testJceAesCtrCryptoCodec() throws Exception {
- if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
- LOG.warn("Skipping since test was not run with -Pnative flag");
- Assume.assumeTrue(false);
- }
+ GenericTestUtils.assumeInNativeProfile();
if (!NativeCodeLoader.buildSupportsOpenssl()) {
LOG.warn("Skipping test since openSSL library not loaded");
Assume.assumeTrue(false);
@@ -91,10 +89,7 @@ public class TestCryptoCodec {
@Test(timeout=120000)
public void testOpensslAesCtrCryptoCodec() throws Exception {
- if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
- LOG.warn("Skipping since test was not run with -Pnative flag");
- Assume.assumeTrue(false);
- }
+ GenericTestUtils.assumeInNativeProfile();
if (!NativeCodeLoader.buildSupportsOpenssl()) {
LOG.warn("Skipping test since openSSL library not loaded");
Assume.assumeTrue(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06549e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index f64e8dc..257ad5d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.crypto;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.BeforeClass;
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
@@ -25,6 +26,7 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
@BeforeClass
public static void init() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
Configuration conf = new Configuration();
codec = CryptoCodec.getInstance(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06549e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
index 4576642..be4ab92 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.SequenceFile.Writer.Option;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.serializer.JavaSerializationComparator;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -140,6 +141,7 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendRecordCompression() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
Path file = new Path(ROOT_PATH, "testseqappendblockcompr.seq");
fs.delete(file, true);
@@ -173,6 +175,7 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendBlockCompression() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
Path file = new Path(ROOT_PATH, "testseqappendblockcompr.seq");
fs.delete(file, true);
@@ -247,6 +250,8 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendSort() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
+
Path file = new Path(ROOT_PATH, "testseqappendSort.seq");
fs.delete(file, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06549e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 65c18d1..e59e68c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -43,6 +43,7 @@ import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.WriterAppender;
import org.junit.Assert;
+import org.junit.Assume;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -410,4 +411,14 @@ public abstract class GenericTestUtils {
}
}
}
+
+ /**
+ * Skip test if native build profile of Maven is not activated.
+ * Sub-project using this must set 'runningWithNative' property to true
+ * in the definition of native profile in pom.xml.
+ */
+ public static void assumeInNativeProfile() {
+ Assume.assumeTrue(
+ Boolean.valueOf(System.getProperty("runningWithNative", "false")));
+ }
}
[13/13] hadoop git commit: HDFS-8846. Add a unit test for INotify
functionality across a layout version upgrade (Zhe Zhang via Colin P. McCabe)
Posted by vi...@apache.org.
HDFS-8846. Add a unit test for INotify functionality across a layout version upgrade (Zhe Zhang via Colin P. McCabe)
(cherry picked from commit a4d9acc51d1a977bc333da17780c00c72e8546f1)
(cherry picked from commit 9264b7e119efb70fb355904652beeb97e7ad90b9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee830b0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee830b0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee830b0d
Branch: refs/heads/branch-2.7
Commit: ee830b0d279f52287f22f3466c8f226569734e91
Parents: fb06549
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Aug 25 14:09:13 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:30 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/TestDFSInotifyEventInputStream.java | 2 +-
.../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 79 +-------------
.../hadoop/hdfs/TestDFSUpgradeFromImage.java | 107 ++++++++++++++++++-
.../src/test/resources/hadoop-252-dfs-dir.tgz | Bin 0 -> 14112 bytes
5 files changed, 108 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee830b0d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d90ec70..e703aa9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -16,6 +16,9 @@ Release 2.7.2 - UNRELEASED
HDFS-7314. When the DFSClient lease cannot be renewed, abort open-for-write
files rather than the entire DFSClient. (mingma)
+ HDFS-8846. Add a unit test for INotify functionality across a layout
+ version upgrade (Zhe Zhang via Colin P. McCabe)
+
OPTIMIZATIONS
HDFS-8722. Optimize datanode writes for small writes and flushes (kihwal)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee830b0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index ba33bd3..a22fe84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -51,7 +51,7 @@ public class TestDFSInotifyEventInputStream {
private static final Log LOG = LogFactory.getLog(
TestDFSInotifyEventInputStream.class);
- private static EventBatch waitForNextEvents(DFSInotifyEventInputStream eis)
+ public static EventBatch waitForNextEvents(DFSInotifyEventInputStream eis)
throws IOException, MissingEventsException {
EventBatch batch = null;
while ((batch = eis.poll()) == null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee830b0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
index 596cc5f..2a00746 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
@@ -28,20 +28,13 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.IOException;
-import java.nio.file.Files;
-import java.util.List;
import java.util.regex.Pattern;
-import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventBatch;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
@@ -49,21 +42,13 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
-import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
-import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
-import org.apache.hadoop.hdfs.server.namenode.NNStorage;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import static org.apache.hadoop.hdfs.inotify.Event.CreateEvent;
-
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@@ -81,7 +66,7 @@ public class TestDFSUpgrade {
private Configuration conf;
private int testCounter = 0;
private MiniDFSCluster cluster = null;
-
+
/**
* Writes an INFO log message containing the parameters.
*/
@@ -466,68 +451,6 @@ public class TestDFSUpgrade {
}
}
- @Test
- public void testPreserveEditLogs() throws Exception {
- conf = new HdfsConfiguration();
- conf = UpgradeUtilities.initializeStorageStateConf(1, conf);
- String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
- conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);
-
- log("Normal NameNode upgrade", 1);
- File[] created =
- UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
- for (final File createdDir : created) {
- List<String> fileNameList =
- IOUtils.listDirectory(createdDir, EditLogsFilter.INSTANCE);
- for (String fileName : fileNameList) {
- String tmpFileName = fileName + ".tmp";
- File existingFile = new File(createdDir, fileName);
- File tmpFile = new File(createdDir, tmpFileName);
- Files.move(existingFile.toPath(), tmpFile.toPath());
- File newFile = new File(createdDir, fileName);
- Preconditions.checkState(newFile.createNewFile(),
- "Cannot create new edits log file in " + createdDir);
- EditLogFileInputStream in = new EditLogFileInputStream(tmpFile,
- HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID,
- false);
- EditLogFileOutputStream out = new EditLogFileOutputStream(conf, newFile,
- (int)tmpFile.length());
- out.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION + 1);
- FSEditLogOp logOp = in.readOp();
- while (logOp != null) {
- out.write(logOp);
- logOp = in.readOp();
- }
- out.setReadyToFlush();
- out.flushAndSync(true);
- out.close();
- Files.delete(tmpFile.toPath());
- }
- }
-
- cluster = createCluster();
-
- DFSInotifyEventInputStream ieis =
- cluster.getFileSystem().getInotifyEventStream(0);
- EventBatch batch = ieis.poll();
- Event[] events = batch.getEvents();
- assertTrue("Should be able to get transactions before the upgrade.",
- events.length > 0);
- assertEquals(events[0].getEventType(), Event.EventType.CREATE);
- assertEquals(((CreateEvent) events[0]).getPath(), "/TestUpgrade");
- cluster.shutdown();
- UpgradeUtilities.createEmptyDirs(nameNodeDirs);
- }
-
- private enum EditLogsFilter implements FilenameFilter {
- INSTANCE;
-
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith(NNStorage.NameNodeFile.EDITS.getName());
- }
- }
-
public static void main(String[] args) throws Exception {
TestDFSUpgrade t = new TestDFSUpgrade();
TestDFSUpgrade.initialize();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee830b0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
index ad907f6..ffbaa89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
@@ -18,10 +18,6 @@
package org.apache.hadoop.hdfs;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
@@ -40,10 +36,13 @@ import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.inotify.Event;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.test.GenericTestUtils;
@@ -51,6 +50,9 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;
import org.junit.Test;
+import static org.apache.hadoop.hdfs.inotify.Event.CreateEvent;
+import static org.junit.Assert.*;
+
/**
* This tests data transfer protocol handling in the Datanode. It sends
* various forms of wrong data and verifies that Datanode handles it well.
@@ -74,6 +76,7 @@ public class TestDFSUpgradeFromImage {
private static final String HADOOP023_RESERVED_IMAGE =
"hadoop-0.23-reserved.tgz";
private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz";
+ private static final String HADOOP252_IMAGE = "hadoop-252-dfs-dir.tgz";
private static class ReferenceFileInfo {
String path;
@@ -620,4 +623,100 @@ public class TestDFSUpgradeFromImage {
numDataNodes(1).enableManagedDfsDirsRedundancy(false).
manageDataDfsDirs(false), null);
}
+
+ @Test
+ public void testPreserveEditLogs() throws Exception {
+ unpackStorage(HADOOP252_IMAGE, HADOOP_DFS_DIR_TXT);
+ /**
+ * The pre-created image has the following edits:
+ * mkdir /input; mkdir /input/dir1~5
+ * copyFromLocal randome_file_1 /input/dir1
+ * copyFromLocal randome_file_2 /input/dir2
+ * mv /input/dir1/randome_file_1 /input/dir3/randome_file_3
+ * rmdir /input/dir1
+ */
+ Configuration conf = new HdfsConfiguration();
+ conf = UpgradeUtilities.initializeStorageStateConf(1, conf);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
+ .format(false)
+ .manageDataDfsDirs(false)
+ .manageNameDfsDirs(false)
+ .startupOption(StartupOption.UPGRADE)
+ .build();
+ DFSInotifyEventInputStream ieis =
+ cluster.getFileSystem().getInotifyEventStream(0);
+
+ EventBatch batch;
+ Event.CreateEvent ce;
+ Event.RenameEvent re;
+
+ // mkdir /input
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ ce = (Event.CreateEvent) batch.getEvents()[0];
+ assertEquals(ce.getPath(), "/input");
+
+ // mkdir /input/dir1~5
+ for (int i = 1; i <= 5; i++) {
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ ce = (Event.CreateEvent) batch.getEvents()[0];
+ assertEquals(ce.getPath(), "/input/dir" + i);
+ }
+ // copyFromLocal randome_file_1~2 /input/dir1~2
+ for (int i = 1; i <= 2; i++) {
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ if (batch.getEvents()[0].getEventType() != Event.EventType.CREATE) {
+ FSImage.LOG.debug("");
+ }
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+
+ // copyFromLocal randome_file_1 /input/dir1, CLOSE
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CLOSE);
+
+ // copyFromLocal randome_file_1 /input/dir1, CLOSE
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() ==
+ Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir" + i + "/randome_file_" + i);
+ }
+
+ // mv /input/dir1/randome_file_1 /input/dir3/randome_file_3
+ long txIDBeforeRename = batch.getTxid();
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
+
+
+ // rmdir /input/dir1
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.UNLINK);
+ assertEquals(((Event.UnlinkEvent) batch.getEvents()[0]).getPath(),
+ "/input/dir1");
+ long lastTxID = batch.getTxid();
+
+ // Start inotify from the tx before rename /input/dir1/randome_file_1
+ ieis = cluster.getFileSystem().getInotifyEventStream(txIDBeforeRename);
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
+
+ // Try to read beyond available edits
+ ieis = cluster.getFileSystem().getInotifyEventStream(lastTxID + 1);
+ assertNull(ieis.poll());
+
+ cluster.shutdown();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee830b0d/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz
new file mode 100644
index 0000000..4ad3e25
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz differ
[05/13] hadoop git commit: MAPREDUCE-5649. Reduce cannot use more
than 2G memory for the final merge. Contributed by Gera Shegalov (cherry
picked from commit 7dc3c1203d1ab14c09d0aaf0869a5bcdfafb0a5a)
Posted by vi...@apache.org.
MAPREDUCE-5649. Reduce cannot use more than 2G memory for the final merge. Contributed by Gera Shegalov
(cherry picked from commit 7dc3c1203d1ab14c09d0aaf0869a5bcdfafb0a5a)
(cherry picked from commit 87c2d915f1cc799cb4020c945c04d3ecb82ee963)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ea42b84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ea42b84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ea42b84
Branch: refs/heads/branch-2.7
Commit: 4ea42b84b7b3adaa0be5e857cc5f996d5d8a98bf
Parents: afa8189
Author: Jason Lowe <jl...@apache.org>
Authored: Mon May 4 19:02:39 2015 +0000
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:00:00 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../mapreduce/task/reduce/MergeManagerImpl.java | 47 +++++++++++---------
.../mapreduce/task/reduce/TestMergeManager.java | 29 ++++++++++++
3 files changed, 57 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ea42b84/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index efa0f91..63b6129 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -27,6 +27,9 @@ Release 2.7.2 - UNRELEASED
MAPREDUCE-6474. ShuffleHandler can possibly exhaust nodemanager file
descriptors (Kuhu Shukla via jlowe)
+ MAPREDUCE-5649. Reduce cannot use more than 2G memory for the final merge
+ (Gera Shegalov via jlowe)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ea42b84/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
index a4b1aa8..3699ddd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -93,8 +93,10 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
Set<CompressAwarePath> onDiskMapOutputs = new TreeSet<CompressAwarePath>();
private final OnDiskMerger onDiskMerger;
-
- private final long memoryLimit;
+
+ @VisibleForTesting
+ final long memoryLimit;
+
private long usedMemory;
private long commitMemory;
private final long maxSingleShuffleLimit;
@@ -167,11 +169,10 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
}
// Allow unit tests to fix Runtime memory
- this.memoryLimit =
- (long)(jobConf.getLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
- Math.min(Runtime.getRuntime().maxMemory(), Integer.MAX_VALUE))
- * maxInMemCopyUse);
-
+ this.memoryLimit = (long)(jobConf.getLong(
+ MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
+ Runtime.getRuntime().maxMemory()) * maxInMemCopyUse);
+
this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100);
final float singleShuffleMemoryLimitPercent =
@@ -201,7 +202,7 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
if (this.maxSingleShuffleLimit >= this.mergeThreshold) {
throw new RuntimeException("Invalid configuration: "
- + "maxSingleShuffleLimit should be less than mergeThreshold"
+ + "maxSingleShuffleLimit should be less than mergeThreshold "
+ "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit
+ "mergeThreshold: " + this.mergeThreshold);
}
@@ -667,24 +668,26 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
}
}
- private RawKeyValueIterator finalMerge(JobConf job, FileSystem fs,
- List<InMemoryMapOutput<K,V>> inMemoryMapOutputs,
- List<CompressAwarePath> onDiskMapOutputs
- ) throws IOException {
- LOG.info("finalMerge called with " +
- inMemoryMapOutputs.size() + " in-memory map-outputs and " +
- onDiskMapOutputs.size() + " on-disk map-outputs");
-
+ @VisibleForTesting
+ final long getMaxInMemReduceLimit() {
final float maxRedPer =
- job.getFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0f);
+ jobConf.getFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0f);
if (maxRedPer > 1.0 || maxRedPer < 0.0) {
- throw new IOException(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT +
- maxRedPer);
+ throw new RuntimeException(maxRedPer + ": "
+ + MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT
+ + " must be a float between 0 and 1.0");
}
- int maxInMemReduce = (int)Math.min(
- Runtime.getRuntime().maxMemory() * maxRedPer, Integer.MAX_VALUE);
-
+ return (long)(memoryLimit * maxRedPer);
+ }
+ private RawKeyValueIterator finalMerge(JobConf job, FileSystem fs,
+ List<InMemoryMapOutput<K,V>> inMemoryMapOutputs,
+ List<CompressAwarePath> onDiskMapOutputs
+ ) throws IOException {
+ LOG.info("finalMerge called with " +
+ inMemoryMapOutputs.size() + " in-memory map-outputs and " +
+ onDiskMapOutputs.size() + " on-disk map-outputs");
+ final long maxInMemReduce = getMaxInMemReduceLimit();
// merge config params
Class<K> keyClass = (Class<K>)job.getMapOutputKeyClass();
Class<V> valueClass = (Class<V>)job.getMapOutputValueClass();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ea42b84/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
index 8d6bab9..ef860af 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
@@ -260,4 +260,33 @@ public class TestMergeManager {
}
}
+
+ @Test
+ public void testLargeMemoryLimits() throws Exception {
+ final JobConf conf = new JobConf();
+ // Xmx in production
+ conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
+ 8L * 1024 * 1024 * 1024);
+
+ // M1 = Xmx fraction for map outputs
+ conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f);
+
+ // M2 = max M1 fraction for a single maple output
+ conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.95f);
+
+ // M3 = M1 fraction at which in memory merge is triggered
+ conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 1.0f);
+
+ // M4 = M1 fraction of map outputs remaining in memory for a reduce
+ conf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 1.0f);
+
+ final MergeManagerImpl<Text, Text> mgr = new MergeManagerImpl<Text, Text>(
+ null, conf, mock(LocalFileSystem.class), null, null, null, null, null,
+ null, null, null, null, null, new MROutputFiles());
+ assertTrue("Large shuffle area unusable: " + mgr.memoryLimit,
+ mgr.memoryLimit > Integer.MAX_VALUE);
+ final long maxInMemReduce = mgr.getMaxInMemReduceLimit();
+ assertTrue("Large in-memory reduce area unusable: " + maxInMemReduce,
+ maxInMemReduce > Integer.MAX_VALUE);
+ }
}
[09/13] hadoop git commit: HDFS-7609. Avoid retry cache collision
when Standby NameNode loading edits. Contributed by Ming Ma.
Posted by vi...@apache.org.
HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits. Contributed by Ming Ma.
(cherry picked from commit 7817674a3a4d097b647dd77f1345787dd376d5ea)
(cherry picked from commit 17fb442a4c4e43105374c97fccd68dd966729a19)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c127b44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c127b44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c127b44
Branch: refs/heads/branch-2.7
Commit: 6c127b44ca1aede95e29b81cf1c0479ae84a8c71
Parents: 188096f
Author: Jing Zhao <ji...@apache.org>
Authored: Fri May 29 11:05:13 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:28 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/namenode/FSNamesystem.java | 19 ----------
.../hdfs/server/namenode/NameNodeRpcServer.java | 22 ++++++++++--
.../namenode/ha/TestRetryCacheWithHA.java | 37 ++++++++++++++++++--
4 files changed, 58 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c127b44/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cd5a5ea..d22f592 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -53,6 +53,9 @@ Release 2.7.2 - UNRELEASED
HDFS-8431. hdfs crypto class not found in Windows.
(Anu Engineer via cnauroth)
+ HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits.
+ (Ming Ma via jing9)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c127b44/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 2dda12b..6998e56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1874,7 +1874,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void concat(String target, String [] srcs, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
waitForLoadingFSImage();
HdfsFileStatus stat = null;
boolean success = false;
@@ -2376,7 +2375,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean skipSync = false;
HdfsFileStatus stat = null;
FSPermissionChecker pc = getPermissionChecker();
- checkOperation(OperationCategory.WRITE);
if (blockSize < minBlockSize) {
throw new IOException("Specified block size is less than configured" +
" minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
@@ -2974,7 +2972,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
LocatedBlock lb = null;
HdfsFileStatus stat = null;
FSPermissionChecker pc = getPermissionChecker();
- checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
writeLock();
try {
@@ -3645,7 +3642,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean renameTo(String src, String dst, boolean logRetryCache)
throws IOException {
waitForLoadingFSImage();
- checkOperation(OperationCategory.WRITE);
FSDirRenameOp.RenameOldResult ret = null;
writeLock();
try {
@@ -3671,7 +3667,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean logRetryCache, Options.Rename... options)
throws IOException {
waitForLoadingFSImage();
- checkOperation(OperationCategory.WRITE);
Map.Entry<BlocksMapUpdateInfo, HdfsFileStatus> res = null;
writeLock();
try {
@@ -3708,7 +3703,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean delete(String src, boolean recursive, boolean logRetryCache)
throws IOException {
waitForLoadingFSImage();
- checkOperation(OperationCategory.WRITE);
BlocksMapUpdateInfo toRemovedBlocks = null;
writeLock();
boolean ret = false;
@@ -6332,8 +6326,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String clientName, ExtendedBlock oldBlock, ExtendedBlock newBlock,
DatanodeID[] newNodes, String[] newStorageIDs, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
-
LOG.info("updatePipeline(" + oldBlock.getLocalBlock()
+ ", newGS=" + newBlock.getGenerationStamp()
+ ", newLength=" + newBlock.getNumBytes()
@@ -7370,7 +7362,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void renameSnapshot(
String path, String snapshotOldName, String snapshotNewName,
boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
try {
@@ -7454,7 +7445,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void deleteSnapshot(String snapshotRoot, String snapshotName,
boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
BlocksMapUpdateInfo blocksToBeDeleted = null;
@@ -7682,7 +7672,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
long addCacheDirective(CacheDirectiveInfo directive,
EnumSet<CacheFlag> flags, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
CacheDirectiveInfo effectiveDirective = null;
if (!flags.contains(CacheFlag.FORCE)) {
cacheManager.waitForRescanIfNeeded();
@@ -7713,7 +7702,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void modifyCacheDirective(CacheDirectiveInfo directive,
EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
if (!flags.contains(CacheFlag.FORCE)) {
cacheManager.waitForRescanIfNeeded();
@@ -7740,7 +7728,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
void removeCacheDirective(long id, boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
try {
@@ -7782,7 +7769,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void addCachePool(CachePoolInfo req, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
writeLock();
boolean success = false;
String poolInfoStr = null;
@@ -7806,7 +7792,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
writeLock();
boolean success = false;
try {
@@ -7830,7 +7815,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void removeCachePool(String cachePoolName, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
writeLock();
boolean success = false;
try {
@@ -8028,7 +8012,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String src = srcArg;
HdfsFileStatus resultingStat = null;
checkSuperuserPrivilege();
- checkOperation(OperationCategory.WRITE);
final byte[][] pathComponents =
FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
@@ -8114,7 +8097,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
HdfsFileStatus auditStat = null;
writeLock();
try {
@@ -8162,7 +8144,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void removeXAttr(String src, XAttr xAttr, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
HdfsFileStatus auditStat = null;
writeLock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c127b44/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index c057f3e..dfd51f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -610,7 +610,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("create: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
-
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return (HdfsFileStatus) cacheEntry.getPayload();
@@ -641,6 +641,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
stateChangeLog.debug("*DIR* NameNode.append: file "
+src+" for "+clientName+" at "+clientMachine);
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -810,6 +811,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -854,7 +856,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("rename: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
-
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return true; // Return previous response
@@ -875,6 +877,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void concat(String trg, String[] src) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -900,6 +903,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("rename: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -937,6 +941,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
stateChangeLog.debug("*DIR* Namenode.delete: src=" + src
+ ", recursive=" + recursive);
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return true; // Return previous response
@@ -1221,6 +1226,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void createSymlink(String target, String link, FsPermission dirPerms,
boolean createParent) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1552,6 +1558,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("createSnapshot: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1573,6 +1580,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void deleteSnapshot(String snapshotRoot, String snapshotName)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
metrics.incrDeleteSnapshotOps();
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1611,6 +1619,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
if (snapshotNewName == null || snapshotNewName.isEmpty()) {
throw new IOException("The new snapshot name is null or empty.");
}
+ namesystem.checkOperation(OperationCategory.WRITE);
metrics.incrRenameSnapshotOps();
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1650,6 +1659,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public long addCacheDirective(
CacheDirectiveInfo path, EnumSet<CacheFlag> flags) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion
(retryCache, null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1671,6 +1681,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void modifyCacheDirective(
CacheDirectiveInfo directive, EnumSet<CacheFlag> flags) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1688,6 +1699,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void removeCacheDirective(long id) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1714,6 +1726,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override //ClientProtocol
public void addCachePool(CachePoolInfo info) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1730,6 +1743,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void modifyCachePool(CachePoolInfo info) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1746,6 +1760,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void removeCachePool(String cachePoolName) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1808,6 +1823,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void createEncryptionZone(String src, String keyName)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1839,6 +1855,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1868,6 +1885,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void removeXAttr(String src, XAttr xAttr) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c127b44/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index c0d320c..1c270aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -214,7 +214,8 @@ public class TestRetryCacheWithHA {
abstract class AtMostOnceOp {
private final String name;
final DFSClient client;
-
+ int expectedUpdateCount = 0;
+
AtMostOnceOp(String name, DFSClient client) {
this.name = name;
this.client = client;
@@ -224,6 +225,9 @@ public class TestRetryCacheWithHA {
abstract void invoke() throws Exception;
abstract boolean checkNamenodeBeforeReturn() throws Exception;
abstract Object getResult();
+ int getExpectedCacheUpdateCount() {
+ return expectedUpdateCount;
+ }
}
/** createSnapshot operaiton */
@@ -603,7 +607,7 @@ public class TestRetryCacheWithHA {
class DeleteOp extends AtMostOnceOp {
private final String target;
private boolean deleted;
-
+
DeleteOp(DFSClient client, String target) {
super("delete", client);
this.target = target;
@@ -613,12 +617,14 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(target);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
}
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
deleted = client.delete(target, true);
}
@@ -654,12 +660,14 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(target);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
}
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.createSymlink(target, link, false);
}
@@ -772,11 +780,13 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
dfs.addCachePool(new CachePoolInfo(directive.getPool()));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
result = client.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
}
@@ -818,12 +828,15 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+ expectedUpdateCount++;
id = client.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.modifyCacheDirective(
new CacheDirectiveInfo.Builder().
setId(id).
@@ -874,12 +887,15 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+ expectedUpdateCount++;
id = dfs.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.removeCacheDirective(id);
}
@@ -921,6 +937,7 @@ public class TestRetryCacheWithHA {
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.addCachePool(new CachePoolInfo(pool));
}
@@ -953,11 +970,13 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
client.addCachePool(new CachePoolInfo(pool).setLimit(10l));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.modifyCachePool(new CachePoolInfo(pool).setLimit(99l));
}
@@ -990,11 +1009,13 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
client.addCachePool(new CachePoolInfo(pool));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.removeCachePool(pool);
}
@@ -1029,12 +1050,14 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(src);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
}
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.setXAttr(src, "user.key", "value".getBytes(),
EnumSet.of(XAttrSetFlag.CREATE));
}
@@ -1071,7 +1094,9 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(src);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
+ expectedUpdateCount++;
client.setXAttr(src, "user.key", "value".getBytes(),
EnumSet.of(XAttrSetFlag.CREATE));
}
@@ -1079,6 +1104,7 @@ public class TestRetryCacheWithHA {
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.removeXAttr(src, "user.key");
}
@@ -1315,6 +1341,13 @@ public class TestRetryCacheWithHA {
assertTrue("CacheUpdated on NN0: " + updatedNN0, updatedNN0 > 0);
// Cache updated metrics on NN0 should be >0 since NN1 applied the editlog
assertTrue("CacheUpdated on NN1: " + updatedNN1, updatedNN1 > 0);
+ long expectedUpdateCount = op.getExpectedCacheUpdateCount();
+ if (expectedUpdateCount > 0) {
+ assertEquals("CacheUpdated on NN0: " + updatedNN0, expectedUpdateCount,
+ updatedNN0);
+ assertEquals("CacheUpdated on NN0: " + updatedNN1, expectedUpdateCount,
+ updatedNN1);
+ }
}
/**
[08/13] hadoop git commit: HDFS-8431. hdfs crypto class not found in
Windows. Contributed by Anu Engineer.
Posted by vi...@apache.org.
HDFS-8431. hdfs crypto class not found in Windows. Contributed by Anu Engineer.
(cherry picked from commit 50eeea13000f0c82e0567410f0f8b611248f8c1b)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
(cherry picked from commit 25db34127811fbadb9a698fa3a76e24d426fb0f6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/188096f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/188096f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/188096f1
Branch: refs/heads/branch-2.7
Commit: 188096f1fe3cf88123b3494c2bd05a8b84bc0d4c
Parents: e48bffd
Author: cnauroth <cn...@apache.org>
Authored: Wed May 27 22:54:00 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:27 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd | 8 +++++++-
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188096f1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 268ebb0..cd5a5ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -50,6 +50,9 @@ Release 2.7.2 - UNRELEASED
HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart.
(surendra singh lilhore via Xiaoyu Yao)
+ HDFS-8431. hdfs crypto class not found in Windows.
+ (Anu Engineer via cnauroth)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/188096f1/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index d52f52e..8115349 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -58,7 +58,8 @@ if "%1" == "--loglevel" (
exit /b
)
)
- set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath
+
+ set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto
for %%i in ( %hdfscommands% ) do (
if %hdfs-command% == %%i set hdfscommand=true
)
@@ -174,6 +175,10 @@ goto :eof
set CLASS=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
goto :eof
+:crypto
+ set CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
+ goto :eof
+
@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
:make_command_arguments
if "%1" == "--config" (
@@ -226,6 +231,7 @@ goto :eof
@echo lsSnapshottableDir list all snapshottable dirs owned by the current user
@echo Use -help to see options
@echo cacheadmin configure the HDFS cache
+ @echo crypto configure HDFS encryption zones
@echo mover run a utility to move block replicas across storage types
@echo storagepolicies list/get/set block storage policies
@echo.
[10/13] hadoop git commit: HADOOP-8151. Error handling in snappy
decompressor throws invalid exceptions. Contributed by Matt Foley. (harsh)
Posted by vi...@apache.org.
HADOOP-8151. Error handling in snappy decompressor throws invalid exceptions. Contributed by Matt Foley. (harsh)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1389006 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit ac31d6a4485d7ff9074fb5dade7a6cf5292bb347)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
(cherry picked from commit 55427fb66c6d52ce98b4d68a29b592a734014c28)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ab1a9c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ab1a9c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ab1a9c2
Branch: refs/heads/branch-2.7
Commit: 6ab1a9c2de2463d2048b8c27e8f8cd21d4ad1888
Parents: 6c127b4
Author: Harsh J <ha...@apache.org>
Authored: Sun Sep 23 10:37:52 2012 +0000
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:29 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c | 2 +-
.../src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c | 2 +-
.../org/apache/hadoop/io/compress/snappy/SnappyCompressor.c | 4 ++--
.../org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c | 6 +++---
5 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ab1a9c2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bbd4ef3..144c296 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -47,6 +47,9 @@ Release 2.7.2 - UNRELEASED
HADOOP-12213. Interrupted exception can occur when Client#stop is called.
(Kuhu Shukla via ozawa)
+ HADOOP-8151. Error handling in snappy decompressor throws invalid
+ exceptions. (Matt Foley via harsh)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ab1a9c2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c
index 58544f5..9f14312 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c
@@ -83,7 +83,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
compressed_direct_buf_len = LZ4_compress(uncompressed_bytes, compressed_bytes, uncompressed_direct_buf_len);
if (compressed_direct_buf_len < 0){
- THROW(env, "Ljava/lang/InternalError", "LZ4_compress failed");
+ THROW(env, "java/lang/InternalError", "LZ4_compress failed");
}
(*env)->SetIntField(env, thisj, Lz4Compressor_uncompressedDirectBufLen, 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ab1a9c2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c
index 6570303..2b8c91c 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c
@@ -80,7 +80,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_de
uncompressed_direct_buf_len = LZ4_decompress_safe(compressed_bytes, uncompressed_bytes, compressed_direct_buf_len, uncompressed_direct_buf_len);
if (uncompressed_direct_buf_len < 0) {
- THROW(env, "Ljava/lang/InternalError", "LZ4_uncompress_unknownOutputSize failed.");
+ THROW(env, "java/lang/InternalError", "LZ4_uncompress_unknownOutputSize failed.");
}
(*env)->SetIntField(env, thisj, Lz4Decompressor_compressedDirectBufLen, 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ab1a9c2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
index 65c978b..fe827f0 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
@@ -134,11 +134,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
ret = dlsym_snappy_compress(uncompressed_bytes, uncompressed_direct_buf_len,
compressed_bytes, &buf_len);
if (ret != SNAPPY_OK){
- THROW(env, "Ljava/lang/InternalError", "Could not compress data. Buffer length is too small.");
+ THROW(env, "java/lang/InternalError", "Could not compress data. Buffer length is too small.");
return 0;
}
if (buf_len > JINT_MAX) {
- THROW(env, "Ljava/lang/InternalError", "Invalid return buffer length.");
+ THROW(env, "java/lang/InternalError", "Invalid return buffer length.");
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ab1a9c2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
index 022f2b0..d1fd13c 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
@@ -126,11 +126,11 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompres
ret = dlsym_snappy_uncompress(compressed_bytes, compressed_direct_buf_len,
uncompressed_bytes, &uncompressed_direct_buf_len);
if (ret == SNAPPY_BUFFER_TOO_SMALL){
- THROW(env, "Ljava/lang/InternalError", "Could not decompress data. Buffer length is too small.");
+ THROW(env, "java/lang/InternalError", "Could not decompress data. Buffer length is too small.");
} else if (ret == SNAPPY_INVALID_INPUT){
- THROW(env, "Ljava/lang/InternalError", "Could not decompress data. Input is invalid.");
+ THROW(env, "java/lang/InternalError", "Could not decompress data. Input is invalid.");
} else if (ret != SNAPPY_OK){
- THROW(env, "Ljava/lang/InternalError", "Could not decompress data.");
+ THROW(env, "java/lang/InternalError", "Could not decompress data.");
}
(*env)->SetIntField(env, thisj, SnappyDecompressor_compressedDirectBufLen, 0);
[07/13] hadoop git commit: YARN-3700. Made generic history service
load a number of latest applications according to the parameter or the
configuration. Contributed by Xuan Gong.
Posted by vi...@apache.org.
YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.
(cherry picked from commit 54504133f41e36eaea6bb06c7b9ddb249468ecd7)
(cherry picked from commit 839f81a6326b2f8b3d5183178382c1551b0bc259)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e48bffd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e48bffd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e48bffd0
Branch: refs/heads/branch-2.7
Commit: e48bffd05598ff86ff7d185640f13a134470c300
Parents: d3562b4
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed May 27 16:51:48 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:27 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +++++++
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../src/main/resources/yarn-default.xml | 9 +++++++
.../ApplicationHistoryClientService.java | 2 +-
.../ApplicationHistoryManager.java | 12 ++++++---
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 13 +++++++---
.../TestApplicationHistoryClientService.java | 26 +++++++++++++++++---
...pplicationHistoryManagerOnTimelineStore.java | 12 +++++----
.../hadoop/yarn/server/webapp/AppsBlock.java | 7 ++++++
.../hadoop/yarn/server/webapp/WebServices.java | 17 ++++---------
.../src/site/markdown/TimelineServer.md | 1 +
13 files changed, 84 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e144864..9bd5cb1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -26,6 +26,9 @@ Release 2.7.2 - UNRELEASED
YARN-2801. Add documentation for node labels feature. (Wangda Tan and Naganarasimha
G R via ozawa)
+ YARN-3700. Made generic history service load a number of latest applications
+ according to the parameter or the configuration. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7b71be6..c0517ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1381,6 +1381,15 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
+ /**
+ * Defines the max number of applications could be fetched using
+ * REST API or application history protocol and shown in timeline
+ * server web ui.
+ */
+ public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ APPLICATION_HISTORY_PREFIX + "max-applications";
+ public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
TIMELINE_SERVICE_PREFIX + "store-class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 405d319..bb4be32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -33,6 +33,7 @@ public interface YarnWebParams {
String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner";
String APP_STATE = "app.state";
+ String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index cb3ad97..de6906e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1348,6 +1348,15 @@
</property>
<property>
+ <description>
+ Defines the max number of applications could be fetched using REST API or
+ application history protocol and shown in timeline server web ui.
+ </description>
+ <name>yarn.timeline-service.generic-application-history.max-applications</name>
+ <value>10000</value>
+ </property>
+
+ <property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index e64ca14..7427926 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -186,7 +186,7 @@ public class ApplicationHistoryClientService extends AbstractService implements
IOException {
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
- history.getAllApplications().values()));
+ history.getApplications(request.getLimit()).values()));
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
index 041c31b..c218eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
@@ -51,16 +51,20 @@ public interface ApplicationHistoryManager {
IOException;
/**
- * This method returns all Application {@link ApplicationReport}s
- *
+ * This method returns the given number of Application
+ * {@link ApplicationReport}s.
+ *
+ * @param appsNum
+ *
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
- Map<ApplicationId, ApplicationReport> getAllApplications()
- throws YarnException, IOException;
+ Map<ApplicationId, ApplicationReport>
+ getApplications(long appsNum) throws YarnException,
+ IOException;
/**
* Application can have multiple application attempts
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index c7cf07b..479858f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws IOException {
Map<ApplicationId, ApplicationHistoryData> histData =
historyStore.getAllApplications();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 5f4510b..3d8a19a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -78,6 +79,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager;
private String serverHttpAddress;
+ private long maxLoadedApplications;
public ApplicationHistoryManagerOnTimelineStore(
TimelineDataManager timelineDataManager,
@@ -91,6 +93,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
protected void serviceInit(Configuration conf) throws Exception {
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ maxLoadedApplications =
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
super.serviceInit(conf);
}
@@ -101,12 +106,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities(
- ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
- null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
- UserGroupInformation.getLoginUser());
+ ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
+ null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
+ EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index ba701a1..aba3af8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -57,6 +57,7 @@ public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
private final static int MAX_APPS = 2;
+ private static TimelineDataManager dataManager;
@BeforeClass
public static void setup() throws Exception {
@@ -64,7 +65,7 @@ public class TestApplicationHistoryClientService {
TimelineStore store =
TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
- TimelineDataManager dataManager =
+ dataManager =
new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -169,8 +170,27 @@ public class TestApplicationHistoryClientService {
clientService.getApplications(request);
List<ApplicationReport> appReport = response.getApplicationList();
Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.get(0).getApplicationId());
- Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+
+ // Create a historyManager, and set the max_apps can be loaded
+ // as 1.
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ ApplicationHistoryManagerOnTimelineStore historyManager2 =
+ new ApplicationHistoryManagerOnTimelineStore(dataManager,
+ new ApplicationACLsManager(conf));
+ historyManager2.init(conf);
+ historyManager2.start();
+ @SuppressWarnings("resource")
+ ApplicationHistoryClientService clientService2 =
+ new ApplicationHistoryClientService(historyManager2);
+ response = clientService2.getApplications(request);
+ appReport = response.getApplicationList();
+ Assert.assertNotNull(appReport);
+ Assert.assertTrue(appReport.size() == 1);
+ // Expected to get the appReport for application with appId1
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index fcdafc2..8672953 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -180,8 +180,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue());
- Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
- Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L
+ + app.getApplicationId().getId(), app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 3L
+ + +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
@@ -335,7 +337,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps =
- historyManager.getAllApplications().values();
+ historyManager.getApplications(Long.MAX_VALUE).values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
@@ -472,12 +474,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
entity.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
"test diagnostics info");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index a601975..3152165 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@@ -77,9 +78,15 @@ public class AppsBlock extends HtmlBlock {
UserGroupInformation callerUGI = getCallerUGI();
Collection<ApplicationReport> appReports = null;
+
try {
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance(reqAppStates);
+ String appsNumStr = $(APPS_NUM);
+ if (appsNumStr != null && !appsNumStr.isEmpty()) {
+ long appsNum = Long.parseLong(appsNumStr);
+ request.setLimit(appsNum);
+ }
if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index d0ccd74..3064145 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -75,13 +75,11 @@ public class WebServices {
String startedEnd, String finishBegin, String finishEnd,
Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req);
- long num = 0;
- boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
- long countNum = 0;
+ long countNum = Long.MAX_VALUE;
// set values suitable in case both of begin/end not specified
long sBegin = 0;
@@ -90,7 +88,6 @@ public class WebServices {
long fEnd = Long.MAX_VALUE;
if (count != null && !count.isEmpty()) {
- checkCount = true;
countNum = Long.parseLong(count);
if (countNum <= 0) {
throw new BadRequestException("limit value must be greater then 0");
@@ -151,19 +148,20 @@ public class WebServices {
AppsInfo allApps = new AppsInfo();
Collection<ApplicationReport> appReports = null;
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance();
+ request.setLimit(countNum);
try {
if (callerUGI == null) {
// TODO: the request should take the params like what RMWebServices does
// in YARN-1819.
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
appReports = callerUGI.doAs(
new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
@Override
public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(
- GetApplicationsRequest.newInstance()).getApplicationList();
+ return appBaseProt.getApplications(request).getApplicationList();
}
});
}
@@ -172,10 +170,6 @@ public class WebServices {
}
for (ApplicationReport appReport : appReports) {
- if (checkCount && num == countNum) {
- break;
- }
-
if (checkAppStates &&
!appStates.contains(StringUtils.toLowerCase(
appReport.getYarnApplicationState().toString()))) {
@@ -215,7 +209,6 @@ public class WebServices {
AppInfo app = new AppInfo(appReport);
allApps.add(app);
- num++;
}
return allApps;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index fde654b..edf1ff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -183,6 +183,7 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
| `yarn.timeline-service.handler-thread-count` | Handler thread count to serve the client RPC requests. Defaults to `10`. |
| `yarn.timeline-service.client.max-retries` | The maximum number of retries for attempts to publish data to the timeline service.Defaults to `30`. |
| `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
+| `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |
[02/13] hadoop git commit: YARN-2890. MiniYarnCluster should turn on
timeline service if configured to do so. Contributed by Mit Desai.
Posted by vi...@apache.org.
YARN-2890. MiniYarnCluster should turn on timeline service if configured to do so. Contributed by Mit Desai.
(cherry picked from commit 265ed1fe804743601a8b62cabc1e4dc2ec8e502f)
(cherry picked from commit 55b794e7fa205df655c19bbfe1de99091fa9dc64)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6b7dc24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6b7dc24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6b7dc24
Branch: refs/heads/branch-2.7
Commit: f6b7dc24f0d2f7e600df95d2f85b6af86d3d64f4
Parents: cfb27d7
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Apr 8 14:13:10 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 10:55:23 2015 -0700
----------------------------------------------------------------------
.../jobhistory/TestJobHistoryEventHandler.java | 2 +-
.../mapred/TestMRTimelineEventHandling.java | 52 ++++++++-
hadoop-yarn-project/CHANGES.txt | 4 +-
.../distributedshell/TestDistributedShell.java | 2 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 6 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 115 +++++++++++++++++++
6 files changed, 172 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 5d39f35..2b07efb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -496,7 +496,7 @@ public class TestJobHistoryEventHandler {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index c2ef128..eab9026 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -35,6 +35,52 @@ import org.junit.Test;
public class TestMRTimelineEventHandling {
@Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+
+ /*
+ * Timeline service should not start if the config is set to false
+ * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+ MiniMRYarnCluster cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
+ cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+
+ @Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -42,7 +88,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -96,7 +142,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -133,7 +179,7 @@ public class TestMRTimelineEventHandling {
cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7ec3457..e144864 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -254,7 +254,6 @@ Release 2.7.1 - 2015-07-06
YARN-3850. NM fails to read files from full disks which can lead to
container logs being lost and other issues (Varun Saxena via jlowe)
-
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
@@ -1039,6 +1038,9 @@ Release 2.6.1 - UNRELEASED
BUG FIXES
+ YARN-2890. MiniYarnCluster should turn on timeline service if
+ configured to do so. (Mit Desai via hitesh)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 5e6fa46..967d172 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -85,7 +85,7 @@ public class TestDistributedShell {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1, true);
+ numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 365e0bb..f8b27b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -262,8 +261,9 @@ public class MiniYARNCluster extends CompositeService {
addService(new NodeManagerWrapper(index));
}
- if (enableAHS) {
- addService(new ApplicationHistoryServerWrapper());
+ if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) {
+ addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
new file mode 100644
index 0000000..8a3c9e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestMiniYarnCluster {
+ @Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ int numNodeManagers = 1;
+ int numLocalDirs = 1;
+ int numLogDirs = 1;
+ boolean enableAHS;
+
+ /*
+ * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS flag == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = false;
+ MiniYARNCluster cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == true
+ * and enableAHS == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ enableAHS = false;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS == true
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = true;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+}
[06/13] hadoop git commit: HDFS-8219. setStoragePolicy with folder
behavior is different after cluster restart. (surendra singh lilhore via
Xiaoyu Yao)
Posted by vi...@apache.org.
HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart. (surendra singh lilhore via Xiaoyu Yao)
(cherry picked from commit 0100b155019496d077f958904de7d385697d65d9)
(cherry picked from commit e68e8b3b5cff85bfd8bb5b00b9033f63577856d6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3562b4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3562b4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3562b4e
Branch: refs/heads/branch-2.7
Commit: d3562b4e1dd1c163a94dd33993109cf3aba23ef1
Parents: 4ea42b8
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 5 13:41:14 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:22 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/server/namenode/FSEditLog.java | 2 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 44 ++++++++++++++++++++
3 files changed, 48 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3562b4e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8758cba..268ebb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -47,6 +47,9 @@ Release 2.7.2 - UNRELEASED
HDFS-9033. dfsadmin -metasave prints "NaN" for cache used%.
(Brahma Reddy Battula via aajisaka)
+ HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart.
+ (surendra singh lilhore via Xiaoyu Yao)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3562b4e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 57c5832..d8f18e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -740,7 +740,7 @@ public class FSEditLog implements LogsPurgeable {
.setClientMachine(
newNode.getFileUnderConstructionFeature().getClientMachine())
.setOverwrite(overwrite)
- .setStoragePolicyId(newNode.getStoragePolicyID());
+ .setStoragePolicyId(newNode.getLocalStoragePolicyID());
AclFeature f = newNode.getAclFeature();
if (f != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3562b4e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index ff7b743..fe69c32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -26,6 +26,7 @@ import java.util.*;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
@@ -1247,4 +1248,47 @@ public class TestBlockStoragePolicy {
Assert.assertEquals(StorageType.ARCHIVE, i.next().getKey());
}
}
+
+ public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
+ //HDFS8219
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(REPLICATION)
+ .storageTypes(
+ new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+ .build();
+ cluster.waitActive();
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ try {
+ final String file = "/testScheduleWithinSameNode/file";
+ Path dir = new Path("/testScheduleWithinSameNode");
+ fs.mkdirs(dir);
+ // 2. Set Dir policy
+ fs.setStoragePolicy(dir, "COLD");
+ // 3. Create file
+ final FSDataOutputStream out = fs.create(new Path(file));
+ out.writeChars("testScheduleWithinSameNode");
+ out.close();
+ // 4. Set Dir policy
+ fs.setStoragePolicy(dir, "HOT");
+ HdfsFileStatus status = fs.getClient().getFileInfo(file);
+ // 5. get file policy, it should be parent policy.
+ Assert
+ .assertTrue(
+ "File storage policy should be HOT",
+ status.getStoragePolicy()
+ == HdfsConstants.HOT_STORAGE_POLICY_ID);
+ // 6. restart NameNode for reloading edits logs.
+ cluster.restartNameNode(true);
+ // 7. get file policy, it should be parent policy.
+ status = fs.getClient().getFileInfo(file);
+ Assert
+ .assertTrue(
+ "File storage policy should be HOT",
+ status.getStoragePolicy()
+ == HdfsConstants.HOT_STORAGE_POLICY_ID);
+
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
[03/13] hadoop git commit: HADOOP-11812. Implement listLocatedStatus
for ViewFileSystem to speed up split calculation (gera)
Posted by vi...@apache.org.
HADOOP-11812. Implement listLocatedStatus for ViewFileSystem to speed up split calculation (gera)
(cherry picked from commit 6d2cf9fbbd02482315a091ab07af26e40cc5134f)
(cherry picked from commit 1544c63602089b690e850e0e30af4589513a2371)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3f41f6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3f41f6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3f41f6e
Branch: refs/heads/branch-2.7
Commit: d3f41f6ed8f78db7023fbdf91a107bcdb69d0adf
Parents: f6b7dc2
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Apr 21 11:57:42 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 10:56:15 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/fs/LocatedFileStatus.java | 7 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 10 +-
.../org/apache/hadoop/fs/viewfs/InodeTree.java | 2 +-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 94 ++++++++++---
.../fs/viewfs/ViewFsLocatedFileStatus.java | 136 +++++++++++++++++++
.../fs/viewfs/TestChRootedFileSystem.java | 14 ++
.../fs/viewfs/ViewFileSystemBaseTest.java | 108 +++++++++++----
8 files changed, 327 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 50106a7..9aac0d4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -57,6 +57,9 @@ Release 2.7.1 - 2015-07-06
OPTIMIZATIONS
+ HADOOP-11812. Implement listLocatedStatus for ViewFileSystem to speed up
+ split calculation (gera)
+
BUG FIXES
HADOOP-11868. Invalid user logins trigger large backtraces in server log
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
index 0136894..9e920c5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
@@ -32,6 +32,11 @@ import org.apache.hadoop.fs.permission.FsPermission;
public class LocatedFileStatus extends FileStatus {
private BlockLocation[] locations;
+
+ public LocatedFileStatus() {
+ super();
+ }
+
/**
* Constructor
* @param stat a file status
@@ -43,7 +48,7 @@ public class LocatedFileStatus extends FileStatus {
stat.getBlockSize(), stat.getModificationTime(),
stat.getAccessTime(), stat.getPermission(), stat.getOwner(),
stat.getGroup(), null, stat.getPath(), locations);
- if (isSymlink()) {
+ if (stat.isSymlink()) {
setSymlink(stat.getSymlink());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 9650a37..18e2391 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -37,8 +37,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -240,7 +242,13 @@ class ChRootedFileSystem extends FilterFileSystem {
throws IOException {
return super.listStatus(fullPath(f));
}
-
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
+ throws IOException {
+ return super.listLocatedStatus(fullPath(f));
+ }
+
@Override
public boolean mkdirs(final Path f, final FsPermission permission)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
index ef64831..632f325 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
@@ -363,7 +363,7 @@ abstract class InodeTree<T> {
kind = k;
targetFileSystem = targetFs;
resolvedPath = resolveP;
- remainingPath = remainingP;
+ remainingPath = remainingP;
}
// isInternalDir of path resolution completed within the mount table
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 0f77f47..43fe23f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -45,7 +45,10 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -115,8 +118,7 @@ public class ViewFileSystem extends FileSystem {
*/
private String getUriPath(final Path p) {
checkPath(p);
- String s = makeAbsolute(p).toUri().getPath();
- return s;
+ return makeAbsolute(p).toUri().getPath();
}
private Path makeAbsolute(final Path f) {
@@ -282,7 +284,7 @@ public class ViewFileSystem extends FileSystem {
}
assert(res.remainingPath != null);
return res.targetFileSystem.createNonRecursive(res.remainingPath, permission,
- flags, bufferSize, replication, blockSize, progress);
+ flags, bufferSize, replication, blockSize, progress);
}
@Override
@@ -297,7 +299,7 @@ public class ViewFileSystem extends FileSystem {
}
assert(res.remainingPath != null);
return res.targetFileSystem.create(res.remainingPath, permission,
- overwrite, bufferSize, replication, blockSize, progress);
+ overwrite, bufferSize, replication, blockSize, progress);
}
@@ -328,7 +330,7 @@ public class ViewFileSystem extends FileSystem {
final InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(fs.getPath()), true);
return res.targetFileSystem.getFileBlockLocations(
- new ViewFsFileStatus(fs, res.remainingPath), start, len);
+ new ViewFsFileStatus(fs, res.remainingPath), start, len);
}
@Override
@@ -340,24 +342,42 @@ public class ViewFileSystem extends FileSystem {
return res.targetFileSystem.getFileChecksum(res.remainingPath);
}
- @Override
- public FileStatus getFileStatus(final Path f) throws AccessControlException,
- FileNotFoundException, IOException {
- InodeTree.ResolveResult<FileSystem> res =
- fsState.resolve(getUriPath(f), true);
-
- // FileStatus#getPath is a fully qualified path relative to the root of
+
+ private static FileStatus fixFileStatus(FileStatus orig,
+ Path qualified) throws IOException {
+ // FileStatus#getPath is a fully qualified path relative to the root of
// target file system.
// We need to change it to viewfs URI - relative to root of mount table.
-
+
// The implementors of RawLocalFileSystem were trying to be very smart.
- // They implement FileStatus#getOwener lazily -- the object
+ // They implement FileStatus#getOwner lazily -- the object
// returned is really a RawLocalFileSystem that expect the
// FileStatus#getPath to be unchanged so that it can get owner when needed.
- // Hence we need to interpose a new ViewFileSystemFileStatus that
+ // Hence we need to interpose a new ViewFileSystemFileStatus that
// works around.
+ if ("file".equals(orig.getPath().toUri().getScheme())) {
+ orig = wrapLocalFileStatus(orig, qualified);
+ }
+
+ orig.setPath(qualified);
+ return orig;
+ }
+
+ private static FileStatus wrapLocalFileStatus(FileStatus orig,
+ Path qualified) {
+ return orig instanceof LocatedFileStatus
+ ? new ViewFsLocatedFileStatus((LocatedFileStatus)orig, qualified)
+ : new ViewFsFileStatus(orig, qualified);
+ }
+
+
+ @Override
+ public FileStatus getFileStatus(final Path f) throws AccessControlException,
+ FileNotFoundException, IOException {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(f), true);
FileStatus status = res.targetFileSystem.getFileStatus(res.remainingPath);
- return new ViewFsFileStatus(status, this.makeQualified(f));
+ return fixFileStatus(status, this.makeQualified(f));
}
@Override
@@ -378,19 +398,51 @@ public class ViewFileSystem extends FileSystem {
if (!res.isInternalDir()) {
// We need to change the name in the FileStatus as described in
// {@link #getFileStatus }
- ChRootedFileSystem targetFs;
- targetFs = (ChRootedFileSystem) res.targetFileSystem;
int i = 0;
for (FileStatus status : statusLst) {
- String suffix = targetFs.stripOutRoot(status.getPath());
- statusLst[i++] = new ViewFsFileStatus(status, this.makeQualified(
- suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix)));
+ statusLst[i++] = fixFileStatus(status,
+ getChrootedPath(res, status, f));
}
}
return statusLst;
}
@Override
+ public RemoteIterator<LocatedFileStatus>listLocatedStatus(final Path f,
+ final PathFilter filter) throws FileNotFoundException, IOException {
+ final InodeTree.ResolveResult<FileSystem> res = fsState
+ .resolve(getUriPath(f), true);
+ final RemoteIterator<LocatedFileStatus> statusIter = res.targetFileSystem
+ .listLocatedStatus(res.remainingPath);
+
+ if (res.isInternalDir()) {
+ return statusIter;
+ }
+
+ return new RemoteIterator<LocatedFileStatus>() {
+ @Override
+ public boolean hasNext() throws IOException {
+ return statusIter.hasNext();
+ }
+
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ final LocatedFileStatus status = statusIter.next();
+ return (LocatedFileStatus)fixFileStatus(status,
+ getChrootedPath(res, status, f));
+ }
+ };
+ }
+
+ private Path getChrootedPath(InodeTree.ResolveResult<FileSystem> res,
+ FileStatus status, Path f) throws IOException {
+ final String suffix = ((ChRootedFileSystem)res.targetFileSystem)
+ .stripOutRoot(status.getPath());
+ return this.makeQualified(
+ suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix));
+ }
+
+ @Override
public boolean mkdirs(final Path dir, final FsPermission permission)
throws IOException {
InodeTree.ResolveResult<FileSystem> res =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFsLocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFsLocatedFileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFsLocatedFileStatus.java
new file mode 100644
index 0000000..347a809
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFsLocatedFileStatus.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+
+import java.io.IOException;
+
+class ViewFsLocatedFileStatus extends LocatedFileStatus {
+ final LocatedFileStatus myFs;
+ Path modifiedPath;
+
+ ViewFsLocatedFileStatus(LocatedFileStatus locatedFileStatus, Path path) {
+ myFs = locatedFileStatus;
+ modifiedPath = path;
+ }
+
+ @Override
+ public long getLen() {
+ return myFs.getLen();
+ }
+
+ @Override
+ public boolean isFile() {
+ return myFs.isFile();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return myFs.isDirectory();
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public boolean isDir() {
+ return myFs.isDirectory();
+ }
+
+ @Override
+ public boolean isSymlink() {
+ return myFs.isSymlink();
+ }
+
+ @Override
+ public long getBlockSize() {
+ return myFs.getBlockSize();
+ }
+
+ @Override
+ public short getReplication() {
+ return myFs.getReplication();
+ }
+
+ @Override
+ public long getModificationTime() {
+ return myFs.getModificationTime();
+ }
+
+ @Override
+ public long getAccessTime() {
+ return myFs.getAccessTime();
+ }
+
+ @Override
+ public FsPermission getPermission() {
+ return myFs.getPermission();
+ }
+
+ @Override
+ public String getOwner() {
+ return myFs.getOwner();
+ }
+
+ @Override
+ public String getGroup() {
+ return myFs.getGroup();
+ }
+
+ @Override
+ public Path getPath() {
+ return modifiedPath;
+ }
+
+ @Override
+ public void setPath(final Path p) {
+ modifiedPath = p;
+ }
+
+ @Override
+ public Path getSymlink() throws IOException {
+ return myFs.getSymlink();
+ }
+
+ @Override
+ public void setSymlink(Path p) {
+ myFs.setSymlink(p);
+ }
+
+ @Override
+ public BlockLocation[] getBlockLocations() {
+ return myFs.getBlockLocations();
+ }
+
+ @Override
+ public int compareTo(Object o) {
+ return super.compareTo(o);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
index e8d4656..a13ee8d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
@@ -395,6 +395,20 @@ public class TestChRootedFileSystem {
verify(mockFs).getAclStatus(rawPath);
}
+ @Test
+ public void testListLocatedFileStatus() throws IOException {
+ final Path mockMount = new Path("mockfs://foo/user");
+ final Path mockPath = new Path("/usermock");
+ final Configuration conf = new Configuration();
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+ ConfigUtil.addLink(conf, mockPath.toString(), mockMount.toUri());
+ FileSystem vfs = FileSystem.get(URI.create("viewfs:///"), conf);
+ vfs.listLocatedStatus(mockPath);
+ final FileSystem mockFs = ((MockFileSystem)mockMount.getFileSystem(conf))
+ .getRawFileSystem();
+ verify(mockFs).listLocatedStatus(new Path(mockMount.toUri().getPath()));
+ }
+
static class MockFileSystem extends FilterFileSystem {
MockFileSystem() {
super(mock(FileSystem.class));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f41f6e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index a324556..18769c2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
@@ -36,9 +37,11 @@ import static org.junit.Assert.assertFalse;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.ConfigUtil;
@@ -125,7 +128,7 @@ public class ViewFileSystemBaseTest {
void setupMountPoints() {
ConfigUtil.addLink(conf, "/targetRoot", targetTestRoot.toUri());
- ConfigUtil.addLink(conf, "/user", new Path(targetTestRoot,"user").toUri());
+ ConfigUtil.addLink(conf, "/user", new Path(targetTestRoot, "user").toUri());
ConfigUtil.addLink(conf, "/user2", new Path(targetTestRoot,"user").toUri());
ConfigUtil.addLink(conf, "/data", new Path(targetTestRoot,"data").toUri());
ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
@@ -133,9 +136,9 @@ public class ViewFileSystemBaseTest {
ConfigUtil.addLink(conf, "/internalDir/internalDir2/linkToDir3",
new Path(targetTestRoot,"dir3").toUri());
ConfigUtil.addLink(conf, "/danglingLink",
- new Path(targetTestRoot,"missingTarget").toUri());
+ new Path(targetTestRoot, "missingTarget").toUri());
ConfigUtil.addLink(conf, "/linkToAFile",
- new Path(targetTestRoot,"aFile").toUri());
+ new Path(targetTestRoot, "aFile").toUri());
}
@Test
@@ -204,19 +207,28 @@ public class ViewFileSystemBaseTest {
fsView.makeQualified(new Path("/foo/bar")));
}
-
- /**
- * Test modify operations (create, mkdir, delete, etc)
+ @Test
+ public void testLocatedOperationsThroughMountLinks() throws IOException {
+ testOperationsThroughMountLinksInternal(true);
+ }
+
+ @Test
+ public void testOperationsThroughMountLinks() throws IOException {
+ testOperationsThroughMountLinksInternal(false);
+ }
+
+ /**
+ * Test modify operations (create, mkdir, delete, etc)
* on the mount file system where the pathname references through
* the mount points. Hence these operation will modify the target
* file system.
- *
+ *
* Verify the operation via mountfs (ie fSys) and *also* via the
* target file system (ie fSysLocal) that the mount link points-to.
*/
- @Test
- public void testOperationsThroughMountLinks() throws IOException {
- // Create file
+ private void testOperationsThroughMountLinksInternal(boolean located)
+ throws IOException {
+ // Create file
fileSystemTestHelper.createFile(fsView, "/user/foo");
Assert.assertTrue("Created file should be type file",
fsView.isFile(new Path("/user/foo")));
@@ -329,7 +341,8 @@ public class ViewFileSystemBaseTest {
fsView.mkdirs(new Path("/targetRoot/dirFoo"));
Assert.assertTrue(fsView.exists(new Path("/targetRoot/dirFoo")));
boolean dirFooPresent = false;
- for (FileStatus fileStatus : fsView.listStatus(new Path("/targetRoot/"))) {
+ for (FileStatus fileStatus :
+ listStatusInternal(located, new Path("/targetRoot/"))) {
if (fileStatus.getPath().getName().equals("dirFoo")) {
dirFooPresent = true;
}
@@ -394,9 +407,13 @@ public class ViewFileSystemBaseTest {
i++;
}
}
-
-
-
+
+ @Test
+ public void testLocatedListOnInternalDirsOfMountTable() throws IOException {
+ testListOnInternalDirsOfMountTableInternal(true);
+ }
+
+
/**
* Test "readOps" (e.g. list, listStatus)
* on internal dirs of mount table
@@ -406,15 +423,20 @@ public class ViewFileSystemBaseTest {
// test list on internal dirs of mount table
@Test
public void testListOnInternalDirsOfMountTable() throws IOException {
+ testListOnInternalDirsOfMountTableInternal(false);
+ }
+
+ private void testListOnInternalDirsOfMountTableInternal(boolean located)
+ throws IOException {
// list on Slash
-
- FileStatus[] dirPaths = fsView.listStatus(new Path("/"));
+
+ FileStatus[] dirPaths = listStatusInternal(located, new Path("/"));
FileStatus fs;
verifyRootChildren(dirPaths);
// list on internal dir
- dirPaths = fsView.listStatus(new Path("/internalDir"));
+ dirPaths = listStatusInternal(located, new Path("/internalDir"));
Assert.assertEquals(2, dirPaths.length);
fs = fileSystemTestHelper.containsPath(fsView, "/internalDir/internalDir2", dirPaths);
@@ -452,13 +474,26 @@ public class ViewFileSystemBaseTest {
@Test
public void testListOnMountTargetDirs() throws IOException {
- FileStatus[] dirPaths = fsView.listStatus(new Path("/data"));
+ testListOnMountTargetDirsInternal(false);
+ }
+
+ @Test
+ public void testLocatedListOnMountTargetDirs() throws IOException {
+ testListOnMountTargetDirsInternal(true);
+ }
+
+ private void testListOnMountTargetDirsInternal(boolean located)
+ throws IOException {
+ final Path dataPath = new Path("/data");
+
+ FileStatus[] dirPaths = listStatusInternal(located, dataPath);
+
FileStatus fs;
Assert.assertEquals(0, dirPaths.length);
// add a file
long len = fileSystemTestHelper.createFile(fsView, "/data/foo");
- dirPaths = fsView.listStatus(new Path("/data"));
+ dirPaths = listStatusInternal(located, dataPath);
Assert.assertEquals(1, dirPaths.length);
fs = fileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths);
Assert.assertNotNull(fs);
@@ -467,7 +502,7 @@ public class ViewFileSystemBaseTest {
// add a dir
fsView.mkdirs(fileSystemTestHelper.getTestRootPath(fsView, "/data/dirX"));
- dirPaths = fsView.listStatus(new Path("/data"));
+ dirPaths = listStatusInternal(located, dataPath);
Assert.assertEquals(2, dirPaths.length);
fs = fileSystemTestHelper.containsPath(fsView, "/data/foo", dirPaths);
Assert.assertNotNull(fs);
@@ -476,7 +511,23 @@ public class ViewFileSystemBaseTest {
Assert.assertNotNull(fs);
Assert.assertTrue("Created dir should appear as a dir", fs.isDirectory());
}
-
+
+ private FileStatus[] listStatusInternal(boolean located, Path dataPath) throws IOException {
+ FileStatus[] dirPaths = new FileStatus[0];
+ if (located) {
+ RemoteIterator<LocatedFileStatus> statIter =
+ fsView.listLocatedStatus(dataPath);
+ ArrayList<LocatedFileStatus> tmp = new ArrayList<LocatedFileStatus>(10);
+ while (statIter.hasNext()) {
+ tmp.add(statIter.next());
+ }
+ dirPaths = tmp.toArray(dirPaths);
+ } else {
+ dirPaths = fsView.listStatus(dataPath);
+ }
+ return dirPaths;
+ }
+
@Test
public void testFileStatusOnMountLink() throws IOException {
Assert.assertTrue(fsView.getFileStatus(new Path("/")).isDirectory());
@@ -692,11 +743,21 @@ public class ViewFileSystemBaseTest {
Assert.assertTrue("Created file should be type file",
fsView.isFile(new Path("/user/foo")));
Assert.assertTrue("Target of created file should be type file",
- fsTarget.isFile(new Path(targetTestRoot,"user/foo")));
+ fsTarget.isFile(new Path(targetTestRoot, "user/foo")));
}
@Test
public void testRootReadableExecutable() throws IOException {
+ testRootReadableExecutableInternal(false);
+ }
+
+ @Test
+ public void testLocatedRootReadableExecutable() throws IOException {
+ testRootReadableExecutableInternal(true);
+ }
+
+ private void testRootReadableExecutableInternal(boolean located)
+ throws IOException {
// verify executable permission on root: cd /
//
Assert.assertFalse("In root before cd",
@@ -707,7 +768,8 @@ public class ViewFileSystemBaseTest {
// verify readable
//
- verifyRootChildren(fsView.listStatus(fsView.getWorkingDirectory()));
+ verifyRootChildren(listStatusInternal(located,
+ fsView.getWorkingDirectory()));
// verify permissions
//